Execute Multiple Tests on Water Level Data

This Notebook loads a water level dataset and then uses the Configuration and Stream concepts from QARTOD v2 to run a series of tests.

The steps:

  • Create a Bokeh-based function that can plot the results from any test

  • Create a configuration string and convert that into a Config object

  • Read data from a CSV file into a Pandas dataframe

  • Make a Stream from that dataframe

  • Run tests on the Stream using the Config and put the results in a list

  • Plot all the results with the original data

[1]:
from __future__ import annotations

import numpy as np
import pandas as pd
from bokeh.layouts import gridplot
from bokeh.models import ColumnDataSource
from bokeh.plotting import figure, output_notebook, show

from ioos_qc.config import Config
from ioos_qc.results import collect_results
from ioos_qc.streams import PandasStream

output_notebook()
Loading BokehJS ...
[2]:
def plot_results(data, var_name, collected_results, title, test_name):
    """Plot original data together with quality flags"""
    # create a ColumnDataSource by passing the dataframe of original data
    source = ColumnDataSource(data=data)

    # add flags to the data structure Bokeh uses
    context_result = next(r for r in collected_results if r.stream_id == var_name and r.test == test_name)
    qc_test = context_result.results
    source.data["qc_pass"] = np.ma.masked_where(qc_test != 1, data[var_name])
    source.data["qc_suspect"] = np.ma.masked_where(qc_test != 3, data[var_name])
    source.data["qc_fail"] = np.ma.masked_where(qc_test != 4, data[var_name])
    source.data["qc_notrun"] = np.ma.masked_where(qc_test != 2, data[var_name])

    # set-up the figure
    p1 = figure(x_axis_type="datetime", title=test_name + " : " + title)
    p1.grid.grid_line_alpha = 0.3
    p1.xaxis.axis_label = "Time"
    p1.yaxis.axis_label = "Observation Value"

    p1.line(x="time", y=var_name, source=source, legend_label="obs", color="#A6CEE3")
    p1.scatter(
        x="time",
        y="qc_notrun",
        source=source,
        size=2,
        color="gray",
        alpha=0.2,
        legend_label="qc not run",
    )
    p1.scatter(
        x="time",
        y="qc_pass",
        source=source,
        size=4,
        color="green",
        alpha=0.5,
        legend_label="qc pass",
    )
    p1.scatter(
        x="time",
        y="qc_suspect",
        source=source,
        size=4,
        color="orange",
        alpha=0.7,
        legend_label="qc suspect",
    )
    p1.scatter(
        x="time",
        y="qc_fail",
        source=source,
        size=6,
        color="red",
        alpha=1.0,
        legend_label="qc fail",
    )

    show(gridplot([[p1]], width=800, height=400))

Specify data and QC configuration

[3]:
# QC configuration
# This configuration is used to connect the variable with the corresponding test
# method(s) in the ioos_qc library. See documentation for description of each test and its inputs:
#   https://ioos.github.io/ioos_qc/api/ioos_qc.html#module-ioos_qc.qartod

# This string is in YAML but it could have been JSON too.
config_string = """
contexts:
    - streams:
        sea_surface_height_above_sea_level:
            qartod:
                gross_range_test:
                    fail_span: [-10, 10]
                    suspect_span: [-2, 3]
                flat_line_test:
                    tolerance: 0.001
                    suspect_threshold: 10800
                    fail_threshold: 21600
                rate_of_change_test:
                    threshold: 0.001
                spike_test:
                    suspect_threshold: 0.8
                    fail_threshold: 3
"""

# Create a high level and flexible configuration object
context_configs = Config(config_string)

Load data, run tests and plot results

[4]:
import pandas as pd

url = "https://github.com/ioos/ioos_qc/raw/main/docs/source/examples"
fname = f"{url}/water_level_example.csv"
variable_name = "sea_surface_height_above_sea_level"

data = pd.read_csv(fname, parse_dates=["time"])
data.head()
[4]:
time timestamp longitude latitude z sea_surface_height_above_sea_level
0 2018-09-05 21:00:00+00:00 1536181200 NaN NaN 0 0.4785
1 2018-09-05 22:00:00+00:00 1536184800 NaN NaN 0 0.4420
2 2018-09-05 23:00:00+00:00 1536188400 NaN NaN 0 0.4968
3 2018-09-06 01:00:00+00:00 1536195600 NaN NaN 0 0.5456
4 2018-09-06 02:00:00+00:00 1536199200 NaN NaN 0 0.5761
[5]:
# Convert the data to a Stream (Pandas dataframe to a PandasStream)
pandas_stream = PandasStream(data)

# Pass the run method the config to use
results = pandas_stream.run(context_configs)

# Then collect all the results into a single list
results_list = collect_results(results, how="list")
[6]:
title = "Water Level [MHHW] [m] : Kotzebue, AK"

plot_results(data, variable_name, results_list, title, "gross_range_test")
[7]:
plot_results(data, variable_name, results_list, title, "flat_line_test")
[8]:
plot_results(data, variable_name, results_list, title, "rate_of_change_test")
[9]:
plot_results(data, variable_name, results_list, title, "spike_test")