geomon/voegelsberg/update_daily_tachymeter_observations.py

''' module for importing new observations to existuing datasets '''
from ast import Str
import os
import csv
# import requests
from datetime import datetime
from typing import List
from unicodedata import numeric
import uuid
from pyproj import Transformer
# from insert_sensor.wrapper import (Offering, FoI, Procedure)
from sqlalchemy.orm import session
from sqlalchemy import asc, desc
from db.models import (
    Observation,
    create_pg_session,
    Dataset,
    Procedure,
    Phenomenon,
    Platform
)


def main():
    ''' main method '''
    # open postgresql session
    pg_session: session = create_pg_session()
    # platform_sta_identifier = "voegelsberg_tachymeter"
    observations_file_path: str = os.environ.get("VOEGELSBERG_DATA_PATH")
    count_new_inserted_observations: numeric = 0
    list_of_not_existent_sensors: List[str] = []

    # with open('voegelsberg/data.txt', 'rt', encoding="utf-8") as csvfile:
    with open(observations_file_path, 'rt', encoding="utf-8") as csvfile:
        spamreader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
        for row in spamreader:
            # print(row)

            sensor: str = row['Punktnummer']
            pg_query = pg_session.query(Dataset) \
                .join(Procedure) \
                .join(Phenomenon) \
                .filter(Procedure.sta_identifier == sensor)
            location_dataset: Dataset = pg_query.filter(
                Phenomenon.sta_identifier == "TachymeterLocation").first()
            if not location_dataset:
                print("Sensor " + sensor +
                      " ist noch nicht in der Datenbank angelegt!")
                if sensor not in list_of_not_existent_sensors:
                    list_of_not_existent_sensors.append(sensor)

                continue

            zeitstempel = row['Epoche']  # '01.09.2021'
            date_obj = datetime.strptime(
                zeitstempel, '%d.%m.%Y').isoformat()  # '2021-09-01T00:00:00'

            existing_observation: bool = (
                pg_session.query(Observation)
                .filter(Observation.result_time ==
                        date_obj, Observation.fk_dataset_id == location_dataset.id)
                .one_or_none()
            )

            successfully_inserted = False
            # Can we insert this observation?
            if existing_observation is None:
                create_observation(location_dataset, row, pg_session)
                successfully_inserted = True
                count_new_inserted_observations += 1
            else:
                print(f"observation with result time {existing_observation.result_time} "
                      f"for tachymeter {sensor} already exists!")

            # commit new observations:
            if successfully_inserted:
                if not location_dataset.is_published:
                    location_dataset.is_published = 1
                    location_dataset.is_hidden = 0
                    location_dataset.dataset_type = "timeseries"
                    # location_dataset.dataset_type = "trajectory"
                    location_dataset.observation_type = "simple"
                    # location_dataset.value_type = "geometry"
                    location_dataset.value_type = "text"
                pg_session.commit()

        # end for loop sensors end

        if count_new_inserted_observations > 0:
            # after inserting all observations update datetime stamps of datasets
            # for location all datasets of Voegelsberg
            actualize_first_last_observations()

        if len(list_of_not_existent_sensors) > 0:
            sep = ", "
            # joins elements of list1 by ', '
            # and stores in string s
            failed_sensors = sep.join(list_of_not_existent_sensors)
            print("Insert the following sensors an rerun this script: " +
                failed_sensors)

        # close postgresql session
        pg_session.close()


def create_observation(location_dataset: Dataset, data,  pg_session: session):
    ''' create observation in db'''
    # print("Sesnor key exist in JSON data")
    transprojr = Transformer.from_crs(31254, 4326, always_xy=True)
    x_1, y_1, z_1 = (float(data['Y']), float(data['X']), float(data['H']))
    cord_x, cord_y = map(float, transprojr.transform(x_1, y_1))
    print((cord_x, cord_y))  # (11.597409730065536, 47.27196543449542)

    sensor: str = data['Punktnummer']

    zeitstempel = data['Epoche']
    date_obj = datetime.strptime(zeitstempel, '%d.%m.%Y').isoformat()

    # existing_observation: bool = (
    #     pg_session.query(Observation)
    #     .filter(Observation.result_time ==
    #             date_obj, Observation.fk_dataset_id == location_dataset.id)
    #     .one_or_none()
    # )

    # Can we insert this observation?
    # if existing_observation is None:
    # insert bew observation
    new_observation: Observation = Observation()
    # new_observation.id = max_id
    new_observation.sta_identifier = str(uuid.uuid4())
    new_observation.result_time = date_obj
    new_observation.sampling_time_start = new_observation.result_time
    new_observation.sampling_time_end = new_observation.result_time
    new_observation.value_type = "text"
    new_observation.value_geometry = f'POINT({cord_x} {cord_y} {z_1})'
    new_observation.value_text = '{"type":"Point","coordinates":[' + str(
        cord_x) + ',' + str(cord_y) + ',' + str(z_1) + ']}'
    new_observation.fk_dataset_id = location_dataset.id
    pg_session.add(new_observation)
    print(f"new observation with result time {new_observation.result_time} "
          f"for drill hole {sensor} succesfully imported!")
    # return True
    # else:
    #     print(f"observation with result time {existing_observation.result_time} "
    #           f"for tachymeter {sensor} already exists!")
    #     return False


def actualize_first_last_observations():
    ''' iterate through all datasets of Voregelsberg project area
        and actualize last and first corresponding observations'''
    pg_session: session = create_pg_session()
    platform_sta_identifier = "voegelsberg_tachymeter"
    # sensor_platform = pg_session.query(Platform.id) \
    #                 .filter(Platform.sta_identifier == platform_sta_identifier) \
    #                 .first()

    voegelsberg_datasets: List[Dataset] = []
    voegelsberg_datasets = pg_session.query(Dataset) \
        .join(Procedure) \
        .join(Phenomenon) \
        .join(Platform) \
        .filter(Platform.sta_identifier == platform_sta_identifier).all()

    for location_dataset in voegelsberg_datasets:
        last_location_observation = pg_session.query(Observation) \
            .filter(Observation.fk_dataset_id == location_dataset.id) \
            .order_by(desc('sampling_time_start')) \
            .first()
        if last_location_observation is not None:
            location_dataset.last_time = last_location_observation.sampling_time_start
            # location_dataset.last_value = last_location_observation.value_quantity
            location_dataset.fk_last_observation_id = last_location_observation.id

        first_location_observation = pg_session.query(Observation) \
            .filter(Observation.fk_dataset_id == location_dataset.id) \
            .order_by(asc('sampling_time_start')) \
            .first()
        if first_location_observation is not None:
            location_dataset.first_time = first_location_observation.sampling_time_start
            # roll_dataset.first_value = first_location_observation.value_quantity
            location_dataset.fk_first_observation_id = first_location_observation.id

        pg_session.commit()


if __name__ == '__main__':
    main()