- update script for observations of voegelsberg tachymeters

voegelsberg/update_daily_tachymeter_observations.py

@@ -0,0 +1,189 @@
+''' module for importing new observations to existuing datasets '''
+from ast import Str
+import os
+import csv
+# import requests
+from datetime import datetime
+from typing import List
+from unicodedata import numeric
+import uuid
+from pyproj import Transformer
+# from insert_sensor.wrapper import (Offering, FoI, Procedure)
+from sqlalchemy.orm import session
+from sqlalchemy import asc, desc
+from db.models import (
+    Observation,
+    create_pg_session,
+    Dataset,
+    Procedure,
+    Phenomenon,
+    Platform
+)
+
+
+def main():
+    ''' main method '''
+    # open postgresql session
+    pg_session: session = create_pg_session()
+    # platform_sta_identifier = "voegelsberg_tachymeter"
+    observations_file_path: str = os.environ.get("VOEGELSBERG_DATA_PATH")
+    count_new_inserted_observations: numeric = 0
+    list_of_not_existent_sensors: List[str] = []
+
+    # with open('voegelsberg/data.txt', 'rt', encoding="utf-8") as csvfile:
+    with open(observations_file_path, 'rt', encoding="utf-8") as csvfile:
+        spamreader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
+        for row in spamreader:
+            # print(row)
+
+            sensor: str = row['Punktnummer']
+            pg_query = pg_session.query(Dataset) \
+                .join(Procedure) \
+                .join(Phenomenon) \
+                .filter(Procedure.sta_identifier == sensor)
+            location_dataset: Dataset = pg_query.filter(
+                Phenomenon.sta_identifier == "TachymeterLocation").first()
+            if not location_dataset:
+                print("Sensor " + sensor +
+                      " ist noch nicht in der Datenbank angelegt!")
+                if sensor not in list_of_not_existent_sensors:
+                    list_of_not_existent_sensors.append(sensor)
+
+                continue
+
+            zeitstempel = row['Epoche']  # '01.09.2021'
+            date_obj = datetime.strptime(
+                zeitstempel, '%d.%m.%Y').isoformat()  # '2021-09-01T00:00:00'
+
+            existing_observation: bool = (
+                pg_session.query(Observation)
+                .filter(Observation.result_time ==
+                        date_obj, Observation.fk_dataset_id == location_dataset.id)
+                .one_or_none()
+            )
+
+            successfully_inserted = False
+            # Can we insert this observation?
+            if existing_observation is None:
+                create_observation(location_dataset, row, pg_session)
+                successfully_inserted = True
+                count_new_inserted_observations += 1
+            else:
+                print(f"observation with result time {existing_observation.result_time} "
+                      f"for tachymeter {sensor} already exists!")
+
+            # commit new observations:
+            if successfully_inserted:
+                if not location_dataset.is_published:
+                    location_dataset.is_published = 1
+                    location_dataset.is_hidden = 0
+                    location_dataset.dataset_type = "timeseries"
+                    # location_dataset.dataset_type = "trajectory"
+                    location_dataset.observation_type = "simple"
+                    # location_dataset.value_type = "geometry"
+                    location_dataset.value_type = "text"
+                pg_session.commit()
+
+        # end for loop sensors end
+
+        if count_new_inserted_observations > 0:
+            # after inserting all observations update datetime stamps of datasets
+            # for location all datasets of Voegelsberg
+            actualize_first_last_observations()
+
+        if len(list_of_not_existent_sensors) > 0:
+            sep = ", "
+            # joins elements of list1 by ', '
+            # and stores in string s
+            failed_sensors = sep.join(list_of_not_existent_sensors)
+            print("Insert the following sensors an rerun this script: " +
+                failed_sensors)
+
+        # close postgresql session
+        pg_session.close()
+
+
+def create_observation(location_dataset: Dataset, data,  pg_session: session):
+    ''' create observation in db'''
+    # print("Sesnor key exist in JSON data")
+    transprojr = Transformer.from_crs(31254, 4326, always_xy=True)
+    x_1, y_1, z_1 = (float(data['Y']), float(data['X']), float(data['H']))
+    cord_x, cord_y = map(float, transprojr.transform(x_1, y_1))
+    print((cord_x, cord_y))  # (11.597409730065536, 47.27196543449542)
+
+    sensor: str = data['Punktnummer']
+
+    zeitstempel = data['Epoche']
+    date_obj = datetime.strptime(zeitstempel, '%d.%m.%Y').isoformat()
+
+    # existing_observation: bool = (
+    #     pg_session.query(Observation)
+    #     .filter(Observation.result_time ==
+    #             date_obj, Observation.fk_dataset_id == location_dataset.id)
+    #     .one_or_none()
+    # )
+
+    # Can we insert this observation?
+    # if existing_observation is None:
+    # insert bew observation
+    new_observation: Observation = Observation()
+    # new_observation.id = max_id
+    new_observation.sta_identifier = str(uuid.uuid4())
+    new_observation.result_time = date_obj
+    new_observation.sampling_time_start = new_observation.result_time
+    new_observation.sampling_time_end = new_observation.result_time
+    new_observation.value_type = "text"
+    new_observation.value_geometry = f'POINT({cord_x} {cord_y} {z_1})'
+    new_observation.value_text = '{"type":"Point","coordinates":[' + str(
+        cord_x) + ',' + str(cord_y) + ',' + str(z_1) + ']}'
+    new_observation.fk_dataset_id = location_dataset.id
+    pg_session.add(new_observation)
+    print(f"new observation with result time {new_observation.result_time} "
+          f"for drill hole {sensor} succesfully imported!")
+    # return True
+    # else:
+    #     print(f"observation with result time {existing_observation.result_time} "
+    #           f"for tachymeter {sensor} already exists!")
+    #     return False
+
+
+def actualize_first_last_observations():
+    ''' iterate through all datasets of Voregelsberg project area
+        and actualize last and first corresponding observations'''
+    pg_session: session = create_pg_session()
+    platform_sta_identifier = "voegelsberg_tachymeter"
+    # sensor_platform = pg_session.query(Platform.id) \
+    #                 .filter(Platform.sta_identifier == platform_sta_identifier) \
+    #                 .first()
+
+    voegelsberg_datasets: List[Dataset] = []
+    voegelsberg_datasets = pg_session.query(Dataset) \
+        .join(Procedure) \
+        .join(Phenomenon) \
+        .join(Platform) \
+        .filter(Platform.sta_identifier == platform_sta_identifier).all()
+
+    for location_dataset in voegelsberg_datasets:
+        last_location_observation = pg_session.query(Observation) \
+            .filter(Observation.fk_dataset_id == location_dataset.id) \
+            .order_by(desc('sampling_time_start')) \
+            .first()
+        if last_location_observation is not None:
+            location_dataset.last_time = last_location_observation.sampling_time_start
+            # location_dataset.last_value = last_location_observation.value_quantity
+            location_dataset.fk_last_observation_id = last_location_observation.id
+
+        first_location_observation = pg_session.query(Observation) \
+            .filter(Observation.fk_dataset_id == location_dataset.id) \
+            .order_by(asc('sampling_time_start')) \
+            .first()
+        if first_location_observation is not None:
+            location_dataset.first_time = first_location_observation.sampling_time_start
+            # roll_dataset.first_value = first_location_observation.value_quantity
+            location_dataset.fk_first_observation_id = first_location_observation.id
+
+        pg_session.commit()
+
+
+if __name__ == '__main__':
+    main()