Setup

from bokeh.plotting import output_notebook

output_notebook()

Loading BokehJS ...

import numpy as np
from bokeh.layouts import gridplot
from bokeh.plotting import figure, show


def plot_results(data, var_name, results, title, test_name):

    time = data["time"]
    obs = data[var_name]
    qc_test = results["qartod"][test_name]

    qc_pass = np.ma.masked_where(qc_test != 1, obs)
    qc_suspect = np.ma.masked_where(qc_test != 3, obs)
    qc_fail = np.ma.masked_where(qc_test != 4, obs)
    qc_notrun = np.ma.masked_where(qc_test != 2, obs)

    p1 = figure(x_axis_type="datetime", title=test_name + " : " + title)
    p1.grid.grid_line_alpha = 0.3
    p1.xaxis.axis_label = "Time"
    p1.yaxis.axis_label = "Observation Value"

    p1.line(time, obs, legend_label="obs", color="#A6CEE3")
    p1.circle(
        time, qc_notrun, size=2, legend_label="qc not run", color="gray", alpha=0.2
    )
    p1.circle(time, qc_pass, size=4, legend_label="qc pass", color="green", alpha=0.5)
    p1.circle(
        time, qc_suspect, size=4, legend_label="qc suspect", color="orange", alpha=0.7
    )
    p1.circle(time, qc_fail, size=6, legend_label="qc fail", color="red", alpha=1.0)

    show(gridplot([[p1]], width=800, height=400))

Specify data and QC configuration

# QC configuration
# For sea water temperature in degrees C
# This configuration is used to call the corresponding method in the ioos_qc library
# See documentation for description of each test and its inputs:
#   https://ioos.github.io/ioos_qc/api/ioos_qc.html#module-ioos_qc.qartod
qc_config = {
    "qartod": {
        "gross_range_test": {"fail_span": [-10, 10], "suspect_span": [-2, 3]},
        "flat_line_test": {
            "tolerance": 0.001,
            "suspect_threshold": 10800,
            "fail_threshold": 21600,
        },
        "rate_of_change_test": {"threshold": 0.001},
        "spike_test": {"suspect_threshold": 0.8, "fail_threshold": 3},
    }
}

Load data, run tests and plot results

import pandas as pd

url = "https://github.com/ioos/ioos_qc/raw/master/docs/source/examples"
fname = f"{url}/water_level_example.csv"
variable_name = "sea_surface_height_above_sea_level"

data = pd.read_csv(fname, parse_dates=["time"])
data.head()

	time	timestamp	longitude	latitude	z	sea_surface_height_above_sea_level
0	2018-09-05 21:00:00+00:00	1536181200	NaN	NaN	0	0.4785
1	2018-09-05 22:00:00+00:00	1536184800	NaN	NaN	0	0.4420
2	2018-09-05 23:00:00+00:00	1536188400	NaN	NaN	0	0.4968
3	2018-09-06 01:00:00+00:00	1536195600	NaN	NaN	0	0.5456
4	2018-09-06 02:00:00+00:00	1536199200	NaN	NaN	0	0.5761

from ioos_qc.config import QcConfig


qc = QcConfig(qc_config)
qc_results = qc.run(inp=data[variable_name], tinp=data["timestamp"], zinp=data["z"])
qc_results

defaultdict(collections.OrderedDict,
            {'qartod': OrderedDict([('gross_range_test',
                           array([1, 1, 1, ..., 1, 1, 1], dtype=uint8)),
                          ('flat_line_test',
                           array([1, 1, 1, ..., 1, 1, 1], dtype=uint8)),
                          ('rate_of_change_test',
                           array([1, 1, 1, ..., 1, 1, 1], dtype=uint8)),
                          ('spike_test',
                           array([2, 1, 1, ..., 1, 1, 2], dtype=uint8))])})

title = "Water Level [MHHW] [m] : Kotzebue, AK"

plot_results(data, variable_name, qc_results, title, "gross_range_test")

plot_results(data, variable_name, qc_results, title, "flat_line_test")

plot_results(data, variable_name, qc_results, title, "rate_of_change_test")

plot_results(data, variable_name, qc_results, title, "spike_test")