geomon/gschliefgraben_piezometer/import_piezometer_observations_hourly.py

'''
Sqlalchemy version: 1.2.15
Python version: 3.7
'''

import json
import os
import uuid
from datetime import datetime
from sqlalchemy.orm import session
from sqlalchemy import asc, desc, func
from dotenv import load_dotenv, find_dotenv
import requests
from db.models import (
    Observation,
    create_pg_session,
    Dataset,
    Procedure,
    Phenomenon,
    Platform,
    Format
)


def main():
    ''' main method '''
    pg_session: session = create_pg_session()
    platform_sta_identifier = "pechgraben_piezometer"
    # sensor = "bohrloch1"
    # sensor_list = os.environ.get('PIEZOMETER_GSCHLIEFGRABEN_SENSORS', [])
    sensor_list = json.loads(os.environ['PIEZOMETER_GSCHLIEFGRABEN_SENSORS'])

    url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'
    params = {}
    headers = {'content-type': 'application/json'}
    resp = requests.get(url=url, params=params, headers=headers)
    data: json = resp.json()  # Check the JSON Response Content documentation below

    for sensor in sensor_list:
        pg_query = pg_session.query(Dataset) \
            .join(Procedure) \
            .join(Phenomenon) \
            .filter(Procedure.sta_identifier == sensor.lower())
        elevation_dataset: Dataset = pg_query.filter(
            Phenomenon.sta_identifier == "Elevation").first()
        if not elevation_dataset:
            print("Sensor " + sensor + " ist noch nicht angelegt!")
            continue
        if not elevation_dataset.is_published:
            elevation_dataset.is_published = 1
            elevation_dataset.is_hidden = 0
            elevation_dataset.dataset_type = "timeseries"
            elevation_dataset.observation_type = "simple"
            elevation_dataset.value_type = "quantity"
            pg_session.commit()

        platform_exists: bool = pg_session.query(Platform.id).filter_by(
            sta_identifier=platform_sta_identifier).scalar() is not None
        # if platform_exists:
        #     sensor_platform = pg_session.query(Platform.id) \
        #         .filter(Platform.sta_identifier == platform_sta_identifier) \
        #         .first()
        #     elevation_dataset.fk_platform_id = sensor_platform.id

        if not platform_exists:
            sensor_platform = Platform()
            max_id = pg_session.query(func.max(Platform.id)).scalar()
            # sensor_platform.id = max_id + 1
            sensor_platform.sta_identifier = platform_sta_identifier.lower()
            sensor_platform.identifier = platform_sta_identifier.lower()
            sensor_platform.name = platform_sta_identifier.lower()
            elevation_dataset.platform = sensor_platform
        else:
            sensor_platform = pg_session.query(Platform.id) \
                .filter(Platform.sta_identifier == platform_sta_identifier) \
                .first()
            elevation_dataset.fk_platform_id = sensor_platform.id

        format_exists: bool = pg_session.query(Format.id).filter_by(
            definition="http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"
        ).scalar() is not None
        if format_exists:
            sensor_format = pg_session.query(Format.id) \
                .filter(Format.definition ==
                        "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement") \
                .first()
            elevation_dataset.fk_format_id = sensor_format.id
        if sensor in data:
            create_observation(elevation_dataset, sensor, data, pg_session)
            pg_session.commit()

        first_elevation_observation = pg_session.query(Observation) \
            .filter(Observation.fk_dataset_id == elevation_dataset.id) \
            .order_by(asc('sampling_time_start')) \
            .first()
        if first_elevation_observation is not None:
            elevation_dataset.first_time = first_elevation_observation.sampling_time_start
            elevation_dataset.first_value = first_elevation_observation.value_quantity
            elevation_dataset.fk_first_observation_id = first_elevation_observation.id
        last_elevation_observation = pg_session.query(Observation) \
            .filter(Observation.fk_dataset_id == elevation_dataset.id) \
            .order_by(desc('sampling_time_start')) \
            .first()
        if last_elevation_observation is not None:
            elevation_dataset.last_time = last_elevation_observation.sampling_time_start
            elevation_dataset.last_value = last_elevation_observation.value_quantity
            elevation_dataset.fk_last_observation_id = last_elevation_observation.id

    pg_session.commit()
    pg_session.close()


def create_observation(elevation_dataset: Dataset,
                       sensor_key: str,
                       data: json,
                       db_session: session):
    ''' create observation in db'''
    # print("Sesnor key exist in JSON data")
    sensor_object = data[sensor_key]
    zeitstempel = sensor_object["zeitstempel"]
    abstich = sensor_object["abstich"]
    date_obj = datetime.strptime(
        zeitstempel, '%Y-%m-%d %H:%M:%S')

    existing_observation: bool = (
        db_session.query(Observation)
        .filter(Observation.result_time ==
                date_obj, Observation.fk_dataset_id == elevation_dataset.id)
        .one_or_none()
    )
    # Can we insert this observation?
    if existing_observation is None:
        # insert bew observation
        new_observation: Observation = Observation()
        # new_observation.id = max_id
        new_observation.sta_identifier = str(uuid.uuid4())
        new_observation.result_time = date_obj
        new_observation.sampling_time_start = new_observation.result_time
        new_observation.sampling_time_end = new_observation.result_time
        new_observation.value_type = "quantity"
        new_observation.value_quantity = abstich
        new_observation.fk_dataset_id = elevation_dataset.id
        db_session.add(new_observation)
        print(f"new observation with result time {new_observation.result_time} "
              f"for drill hole {sensor_key} succesfully imported!")
    else:
        print(f"observation with result time {existing_observation.result_time} "
              f"for drill hole {sensor_key} already exists!")


def test():
    ''' test method '''
    sensor_key = 'bohrloch1'
    url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'

    params = {}
    headers = {'content-type': 'application/json'}

    resp = requests.get(url=url, params=params, headers=headers)
    data = resp.json()  # Check the JSON Response Content documentation below
    # sensor_data = json.dumps(data)
    if sensor_key in data:
        print("Sesnor key exist in JSON data")
        sensor_object = data[sensor_key]
        zeitstempel = sensor_object["zeitstempel"]
        abstich = sensor_object["abstich"]
        date_obj = datetime.strptime(
            zeitstempel, '%Y-%m-%d %H:%M:%S')

        new_observation: Observation = Observation()
        # new_observation.id = max_id
        new_observation.sta_identifier = str(uuid.uuid4())
        new_observation.result_time = date_obj
        new_observation.sampling_time_start = new_observation.result_time
        new_observation.sampling_time_end = new_observation.result_time
        new_observation.value_type = "quantity"
        new_observation.value_quantity = abstich
        # new_observation.fk_dataset_id = dataset.id


if __name__ == "__main__":
    load_dotenv(find_dotenv())
    sensor_list1 = os.environ.get('PIEZOMETER_GSCHLIEFGRABEN_SENSORS', [])
    print(f'sensors: {sensor_list1} .')
    main()