- cron job for hourly importing piezometer data
This commit is contained in:
parent
8db6507252
commit
14d3168d0e
|
|
@ -1,98 +0,0 @@
|
||||||
'''
|
|
||||||
Sqlalchemy version: 1.2.15
|
|
||||||
Python version: 3.7
|
|
||||||
'''
|
|
||||||
|
|
||||||
import os
|
|
||||||
import uuid
|
|
||||||
from sqlalchemy.orm import session
|
|
||||||
from dotenv import load_dotenv, find_dotenv
|
|
||||||
import requests
|
|
||||||
from datetime import datetime
|
|
||||||
from db.models import (
|
|
||||||
Observation,
|
|
||||||
create_pg_session,
|
|
||||||
Dataset,
|
|
||||||
Procedure,
|
|
||||||
Phenomenon,
|
|
||||||
Platform,
|
|
||||||
Format
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
''' main method '''
|
|
||||||
pg_session: session = create_pg_session()
|
|
||||||
platform_sta_identifier = "pechgraben_piezometer"
|
|
||||||
sensor = "bohrloch1"
|
|
||||||
|
|
||||||
pg_query = pg_session.query(Dataset) \
|
|
||||||
.join(Procedure) \
|
|
||||||
.join(Phenomenon) \
|
|
||||||
.filter(Procedure.sta_identifier == sensor.lower())
|
|
||||||
elevation_dataset: Dataset = pg_query.filter(
|
|
||||||
Phenomenon.sta_identifier == "Elevation").first()
|
|
||||||
if not elevation_dataset:
|
|
||||||
print("Sensor " + sensor + " ist noch nicht angelegt!")
|
|
||||||
exit()
|
|
||||||
# if not elevation_dataset.is_published:
|
|
||||||
# elevation_dataset.is_published = 1
|
|
||||||
# elevation_dataset.is_hidden = 0
|
|
||||||
# elevation_dataset.dataset_type = "timeseries"
|
|
||||||
# elevation_dataset.observation_type = "simple"
|
|
||||||
# elevation_dataset.value_type = "text"
|
|
||||||
# pg_session.commit()
|
|
||||||
|
|
||||||
platform_exists: bool = pg_session.query(Platform.id).filter_by(
|
|
||||||
sta_identifier=platform_sta_identifier).scalar() is not None
|
|
||||||
if platform_exists:
|
|
||||||
sensor_platform = pg_session.query(Platform.id) \
|
|
||||||
.filter(Platform.sta_identifier == platform_sta_identifier) \
|
|
||||||
.first()
|
|
||||||
elevation_dataset.fk_platform_id = sensor_platform.id
|
|
||||||
|
|
||||||
format_exists: bool = pg_session.query(Format.id).filter_by(
|
|
||||||
definition="http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"
|
|
||||||
).scalar() is not None
|
|
||||||
if format_exists:
|
|
||||||
sensor_format = pg_session.query(Format.id) \
|
|
||||||
.filter(Format.definition == "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement") \
|
|
||||||
.first()
|
|
||||||
elevation_dataset.fk_format_id = sensor_format.id
|
|
||||||
|
|
||||||
|
|
||||||
def test():
|
|
||||||
''' test method '''
|
|
||||||
sensor_key = 'bohrloch1'
|
|
||||||
url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'
|
|
||||||
|
|
||||||
params = {}
|
|
||||||
headers = {'content-type': 'application/json'}
|
|
||||||
|
|
||||||
resp = requests.get(url=url, params=params, headers=headers)
|
|
||||||
data = resp.json() # Check the JSON Response Content documentation below
|
|
||||||
# sensor_data = json.dumps(data)
|
|
||||||
if sensor_key in data:
|
|
||||||
print("Sesnor key exist in JSON data")
|
|
||||||
sensor_object = data[sensor_key]
|
|
||||||
zeitstempel = sensor_object["zeitstempel"]
|
|
||||||
abstich = sensor_object["abstich"]
|
|
||||||
date_obj = datetime.strptime(
|
|
||||||
zeitstempel, '%Y:%m:%d %H:%M:%S')
|
|
||||||
|
|
||||||
new_observation: Observation = Observation()
|
|
||||||
# new_observation.id = max_id
|
|
||||||
new_observation.sta_identifier = str(uuid.uuid4())
|
|
||||||
new_observation.result_time = date_obj
|
|
||||||
new_observation.sampling_time_start = new_observation.result_time
|
|
||||||
new_observation.sampling_time_end = new_observation.result_time
|
|
||||||
new_observation.value_type = "quantity"
|
|
||||||
new_observation.value_quantity = abstich
|
|
||||||
# new_observation.fk_dataset_id = dataset.id
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
load_dotenv(find_dotenv())
|
|
||||||
sensor_list1 = os.environ.get('GLASFASER_GSCHLIEFGRABEN_SENSORS', [])
|
|
||||||
print(f'sensors: {sensor_list1} .')
|
|
||||||
test()
|
|
||||||
|
|
@ -0,0 +1,140 @@
|
||||||
|
'''
|
||||||
|
Sqlalchemy version: 1.2.15
|
||||||
|
Python version: 3.7
|
||||||
|
'''
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
from sqlalchemy.orm import session
|
||||||
|
from dotenv import load_dotenv, find_dotenv
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
from db.models import (
|
||||||
|
Observation,
|
||||||
|
create_pg_session,
|
||||||
|
Dataset,
|
||||||
|
Procedure,
|
||||||
|
Phenomenon,
|
||||||
|
Platform,
|
||||||
|
Format
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
''' main method '''
|
||||||
|
pg_session: session = create_pg_session()
|
||||||
|
platform_sta_identifier = "pechgraben_piezometer"
|
||||||
|
# sensor = "bohrloch1"
|
||||||
|
# sensor_list = os.environ.get('GLASFASER_GSCHLIEFGRABEN_SENSORS', [])
|
||||||
|
sensor_list = json.loads(os.environ['GLASFASER_GSCHLIEFGRABEN_SENSORS'])
|
||||||
|
|
||||||
|
url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'
|
||||||
|
params = {}
|
||||||
|
headers = {'content-type': 'application/json'}
|
||||||
|
resp = requests.get(url=url, params=params, headers=headers)
|
||||||
|
data: json = resp.json() # Check the JSON Response Content documentation below
|
||||||
|
|
||||||
|
for sensor in sensor_list:
|
||||||
|
pg_query = pg_session.query(Dataset) \
|
||||||
|
.join(Procedure) \
|
||||||
|
.join(Phenomenon) \
|
||||||
|
.filter(Procedure.sta_identifier == sensor.lower())
|
||||||
|
elevation_dataset: Dataset = pg_query.filter(
|
||||||
|
Phenomenon.sta_identifier == "Elevation").first()
|
||||||
|
if not elevation_dataset:
|
||||||
|
print("Sensor " + sensor + " ist noch nicht angelegt!")
|
||||||
|
exit()
|
||||||
|
if not elevation_dataset.is_published:
|
||||||
|
elevation_dataset.is_published = 1
|
||||||
|
elevation_dataset.is_hidden = 0
|
||||||
|
elevation_dataset.dataset_type = "timeseries"
|
||||||
|
elevation_dataset.observation_type = "simple"
|
||||||
|
elevation_dataset.value_type = "text"
|
||||||
|
pg_session.commit()
|
||||||
|
|
||||||
|
platform_exists: bool = pg_session.query(Platform.id).filter_by(
|
||||||
|
sta_identifier=platform_sta_identifier).scalar() is not None
|
||||||
|
if platform_exists:
|
||||||
|
sensor_platform = pg_session.query(Platform.id) \
|
||||||
|
.filter(Platform.sta_identifier == platform_sta_identifier) \
|
||||||
|
.first()
|
||||||
|
elevation_dataset.fk_platform_id = sensor_platform.id
|
||||||
|
|
||||||
|
format_exists: bool = pg_session.query(Format.id).filter_by(
|
||||||
|
definition="http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"
|
||||||
|
).scalar() is not None
|
||||||
|
if format_exists:
|
||||||
|
sensor_format = pg_session.query(Format.id) \
|
||||||
|
.filter(Format.definition == "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement") \
|
||||||
|
.first()
|
||||||
|
elevation_dataset.fk_format_id = sensor_format.id
|
||||||
|
if sensor in data:
|
||||||
|
create_observation(elevation_dataset, sensor, data, pg_session)
|
||||||
|
pg_session.commit()
|
||||||
|
pg_session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_observation(elevation_dataset: Dataset, sensor_key: str, data: json, db_session: session):
|
||||||
|
''' create observation in db'''
|
||||||
|
print("Sesnor key exist in JSON data")
|
||||||
|
sensor_object = data[sensor_key]
|
||||||
|
zeitstempel = sensor_object["zeitstempel"]
|
||||||
|
abstich = sensor_object["abstich"]
|
||||||
|
date_obj = datetime.strptime(
|
||||||
|
zeitstempel, '%Y-%m-%d %H:%M:%S')
|
||||||
|
|
||||||
|
existing_observation: bool = (
|
||||||
|
db_session.query(Observation)
|
||||||
|
.filter(Observation.result_time == date_obj, Observation.fk_dataset_id == elevation_dataset.id)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
# Can we insert this observation?
|
||||||
|
if existing_observation is None:
|
||||||
|
# insert bew observation
|
||||||
|
new_observation: Observation = Observation()
|
||||||
|
# new_observation.id = max_id
|
||||||
|
new_observation.sta_identifier = str(uuid.uuid4())
|
||||||
|
new_observation.result_time = date_obj
|
||||||
|
new_observation.sampling_time_start = new_observation.result_time
|
||||||
|
new_observation.sampling_time_end = new_observation.result_time
|
||||||
|
new_observation.value_type = "quantity"
|
||||||
|
new_observation.value_quantity = abstich
|
||||||
|
db_session.add(new_observation)
|
||||||
|
|
||||||
|
|
||||||
|
def test():
|
||||||
|
''' test method '''
|
||||||
|
sensor_key = 'bohrloch1'
|
||||||
|
url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'
|
||||||
|
|
||||||
|
params = {}
|
||||||
|
headers = {'content-type': 'application/json'}
|
||||||
|
|
||||||
|
resp = requests.get(url=url, params=params, headers=headers)
|
||||||
|
data = resp.json() # Check the JSON Response Content documentation below
|
||||||
|
# sensor_data = json.dumps(data)
|
||||||
|
if sensor_key in data:
|
||||||
|
print("Sesnor key exist in JSON data")
|
||||||
|
sensor_object = data[sensor_key]
|
||||||
|
zeitstempel = sensor_object["zeitstempel"]
|
||||||
|
abstich = sensor_object["abstich"]
|
||||||
|
date_obj = datetime.strptime(
|
||||||
|
zeitstempel, '%Y-%m-%d %H:%M:%S')
|
||||||
|
|
||||||
|
new_observation: Observation = Observation()
|
||||||
|
# new_observation.id = max_id
|
||||||
|
new_observation.sta_identifier = str(uuid.uuid4())
|
||||||
|
new_observation.result_time = date_obj
|
||||||
|
new_observation.sampling_time_start = new_observation.result_time
|
||||||
|
new_observation.sampling_time_end = new_observation.result_time
|
||||||
|
new_observation.value_type = "quantity"
|
||||||
|
new_observation.value_quantity = abstich
|
||||||
|
# new_observation.fk_dataset_id = dataset.id
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
load_dotenv(find_dotenv())
|
||||||
|
sensor_list1 = os.environ.get('GLASFASER_GSCHLIEFGRABEN_SENSORS', [])
|
||||||
|
print(f'sensors: {sensor_list1} .')
|
||||||
|
main()
|
||||||
Loading…
Reference in New Issue
Block a user