''' module for importing new observations to existuing datasets ''' from ast import Str import os import csv # import requests from datetime import datetime from typing import List from unicodedata import numeric import uuid from pyproj import Transformer # from insert_sensor.wrapper import (Offering, FoI, Procedure) from sqlalchemy.orm import session from sqlalchemy import asc, desc from db.models import ( Observation, create_pg_session, Dataset, Procedure, Phenomenon, Platform ) def main(): ''' main method ''' # open postgresql session pg_session: session = create_pg_session() # platform_sta_identifier = "voegelsberg_tachymeter" observations_file_path: str = os.environ.get("VOEGELSBERG_DATA_PATH") count_new_inserted_observations: numeric = 0 list_of_not_existent_sensors: List[str] = [] # with open('voegelsberg/data.txt', 'rt', encoding="utf-8") as csvfile: with open(observations_file_path, 'rt', encoding="utf-8") as csvfile: spamreader = csv.DictReader(csvfile, delimiter=';', quotechar='"') for row in spamreader: # print(row) sensor: str = row['Punktnummer'] pg_query = pg_session.query(Dataset) \ .join(Procedure) \ .join(Phenomenon) \ .filter(Procedure.sta_identifier == sensor) location_dataset: Dataset = pg_query.filter( Phenomenon.sta_identifier == "TachymeterLocation").first() if not location_dataset: print("Sensor " + sensor + " ist noch nicht in der Datenbank angelegt!") if sensor not in list_of_not_existent_sensors: list_of_not_existent_sensors.append(sensor) continue zeitstempel = row['Epoche'] # '01.09.2021' date_obj = datetime.strptime( zeitstempel, '%d.%m.%Y').isoformat() # '2021-09-01T00:00:00' existing_observation: bool = ( pg_session.query(Observation) .filter(Observation.result_time == date_obj, Observation.fk_dataset_id == location_dataset.id) .one_or_none() ) successfully_inserted = False # Can we insert this observation? if existing_observation is None: create_observation(location_dataset, row, pg_session) successfully_inserted = True count_new_inserted_observations += 1 else: print(f"observation with result time {existing_observation.result_time} " f"for tachymeter {sensor} already exists!") # commit new observations: if successfully_inserted: if not location_dataset.is_published: location_dataset.is_published = 1 location_dataset.is_hidden = 0 location_dataset.dataset_type = "timeseries" # location_dataset.dataset_type = "trajectory" location_dataset.observation_type = "simple" # location_dataset.value_type = "geometry" location_dataset.value_type = "text" pg_session.commit() # end for loop sensors end if count_new_inserted_observations > 0: # after inserting all observations update datetime stamps of datasets # for location all datasets of Voegelsberg actualize_first_last_observations() if len(list_of_not_existent_sensors) > 0: sep = ", " # joins elements of list1 by ', ' # and stores in string s failed_sensors = sep.join(list_of_not_existent_sensors) print("Insert the following sensors an rerun this script: " + failed_sensors) # close postgresql session pg_session.close() def create_observation(location_dataset: Dataset, data, pg_session: session): ''' create observation in db''' # print("Sesnor key exist in JSON data") transprojr = Transformer.from_crs(31254, 4326, always_xy=True) x_1, y_1, z_1 = (float(data['Y']), float(data['X']), float(data['H'])) cord_x, cord_y = map(float, transprojr.transform(x_1, y_1)) print((cord_x, cord_y)) # (11.597409730065536, 47.27196543449542) sensor: str = data['Punktnummer'] zeitstempel = data['Epoche'] date_obj = datetime.strptime(zeitstempel, '%d.%m.%Y').isoformat() # existing_observation: bool = ( # pg_session.query(Observation) # .filter(Observation.result_time == # date_obj, Observation.fk_dataset_id == location_dataset.id) # .one_or_none() # ) # Can we insert this observation? # if existing_observation is None: # insert bew observation new_observation: Observation = Observation() # new_observation.id = max_id new_observation.sta_identifier = str(uuid.uuid4()) new_observation.result_time = date_obj new_observation.sampling_time_start = new_observation.result_time new_observation.sampling_time_end = new_observation.result_time new_observation.value_type = "text" new_observation.value_geometry = f'POINT({cord_x} {cord_y} {z_1})' new_observation.value_text = '{"type":"Point","coordinates":[' + str( cord_x) + ',' + str(cord_y) + ',' + str(z_1) + ']}' new_observation.fk_dataset_id = location_dataset.id pg_session.add(new_observation) print(f"new observation with result time {new_observation.result_time} " f"for drill hole {sensor} succesfully imported!") # return True # else: # print(f"observation with result time {existing_observation.result_time} " # f"for tachymeter {sensor} already exists!") # return False def actualize_first_last_observations(): ''' iterate through all datasets of Voregelsberg project area and actualize last and first corresponding observations''' pg_session: session = create_pg_session() platform_sta_identifier = "voegelsberg_tachymeter" # sensor_platform = pg_session.query(Platform.id) \ # .filter(Platform.sta_identifier == platform_sta_identifier) \ # .first() voegelsberg_datasets: List[Dataset] = [] voegelsberg_datasets = pg_session.query(Dataset) \ .join(Procedure) \ .join(Phenomenon) \ .join(Platform) \ .filter(Platform.sta_identifier == platform_sta_identifier).all() for location_dataset in voegelsberg_datasets: last_location_observation = pg_session.query(Observation) \ .filter(Observation.fk_dataset_id == location_dataset.id) \ .order_by(desc('sampling_time_start')) \ .first() if last_location_observation is not None: location_dataset.last_time = last_location_observation.sampling_time_start # location_dataset.last_value = last_location_observation.value_quantity location_dataset.fk_last_observation_id = last_location_observation.id first_location_observation = pg_session.query(Observation) \ .filter(Observation.fk_dataset_id == location_dataset.id) \ .order_by(asc('sampling_time_start')) \ .first() if first_location_observation is not None: location_dataset.first_time = first_location_observation.sampling_time_start # roll_dataset.first_value = first_location_observation.value_quantity location_dataset.fk_first_observation_id = first_location_observation.id pg_session.commit() if __name__ == '__main__': main()