- add hourly cron job for importing piezeometer data for sensor bohrloch1-6

This commit is contained in:
Arno Kaimbacher 2022-03-16 16:37:32 +01:00
parent 14d3168d0e
commit b4d7ce40b3
6 changed files with 90 additions and 70 deletions

View File

@ -40,12 +40,18 @@ def create_pg_session() -> sessionmaker:
# Base.metadata.create_all(engine)
return _session
platform_seq = Sequence('platform_seq', schema="gba") # define sequence explicitly
class Platform(Base):
""" Platform class """
__tablename__ = 'platform'
__table_args__ = {"schema": "gba"}
id = Column('platform_id', Integer, primary_key=True)
# id = Column('platform_id', Integer, primary_key=True)
id = Column('platform_id',
Integer,
platform_seq,
primary_key=True,
server_default=platform_seq.next_value())
identifier = Column('identifier', String)
sta_identifier = Column('sta_identifier', String)
name = Column('name', String)

View File

@ -1,4 +1,4 @@
<sml:PhysicalSystem gml:id={off_name} xmlns:swes=\"http://www.opengis.net/swes/2.0\"
<sml:PhysicalSystem gml:id={gml_id} xmlns:swes=\"http://www.opengis.net/swes/2.0\"
xmlns:sos=\"http://www.opengis.net/sos/2.0\"
xmlns:swe=\"http://www.opengis.net/swe/2.0\"
xmlns:sml=\"http://www.opengis.net/sensorml/2.0\"
@ -65,15 +65,14 @@
</sams:SF_SpatialSamplingFeature>
</sml:feature>
</sml:FeatureList>
</sml:featuresOfInterest>
</sml:featuresOfInterest>
<sml:outputs>
<sml:OutputList>
<sml:output name=\"HumanVisualPerception\">
<swe:DataRecord>
<swe:field name=\"HumanVisualPerception\">
<swe:Text definition=\"HumanVisualPerception\"/>
</swe:field>
</swe:DataRecord>
<sml:output name=\"Elevation\">
<swe:Quantity definition=\"Elevation\">
<swe:label>Elevation</swe:label>
<swe:uom code=\"m\"/>
</swe:Quantity>
</sml:output>
</sml:OutputList>
</sml:outputs>

View File

@ -37,62 +37,32 @@ def main():
"""
sos_url = 'https://geomon.geologie.ac.at/52n-sos-webapp/service'
# Gschliefgraben Glasfaser
# offering = Offering(
# "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
# "inclino1_02",
# "Inklinometer inclino1_02, Gschliefgraben Glasfaser"
# )
# procedure = Procedure( "inclino1_02","inclino1_02")
# foi = FoI("degree", "m", (13.774966, 47.910849, 0.0),
# "inclino1-glasfaser-gschliefgraben",
# "Glasfaser Untersuchungen am Gschliefgraben (Gmunden)")
# offering = Offering(
# "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
# "inclino1_05",
# "Inklinometer inclino1_05, Gschliefgraben Glasfaser"
# )
# procedure = Procedure("inclino1_05", "inclino1_05")
# foi = FoI("degree", "m", (13.774966, 47.910849, 0.0),
# "inclino1-glasfaser-gschliefgraben",
# "Glasfaser Untersuchungen am Gschliefgraben (Gmunden)")
# offering = Offering(
# "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
# "inclino1_14",
# "Inklinometer inclino1_14, Gschliefgraben Glasfaser"
# )
# procedure = Procedure("inclino1_14", "inclino1_14")
# foi = FoI("degree", "m", (13.774966, 47.910849, 0.0),
# "inclino1-glasfaser-gschliefgraben",
# "Glasfaser Untersuchungen am Gschliefgraben (Gmunden)")
# Gschliefgraben Piezometer
offering = Offering(
"https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
"inclino1_06",
"Inklinometer inclino1_06, Gschliefgraben Glasfaser"
"bohrloch",
"Bohrlöcher, Gschliefgraben Piezometer"
)
procedure = Procedure("inclino1_06", "inclino1_06")
procedure = Procedure("bohrloch1", "bohrloch1")
foi = FoI("degree", "m", (13.774966, 47.910849, 0.0),
"inclino1-glasfaser-gschliefgraben",
"Glasfaser Untersuchungen am Gschliefgraben (Gmunden)")
"bohrloch1-glasfaser-gschliefgraben",
"Piezometer1 am Gschliefgraben")
sensor_type = SensorType("inclinometer")
sensor_type = SensorType("piezometer")
post_data = insert_sensor(offering, procedure, foi, sensor_type)
print(post_data)
headers = {'Accept': 'application/json'}
request = requests.post(sos_url, headers=headers, json=post_data)
print(request.text)
# {
# "request" : "InsertSensor",
# "version" : "2.0.0",
# "service" : "SOS",
# "assignedProcedure" : "inclino1_14",
# "assignedOffering" : "inclino1_14"
# }
# {
# "request" : "InsertSensor",
# "version" : "2.0.0",
# "service" : "SOS",
# "assignedProcedure" : "bohrloch1",
# "assignedOffering" : "bohrloch"
# }
def insert_sensor(offering, procedure, foi, sensor_type):
@ -109,7 +79,7 @@ def insert_sensor(offering, procedure, foi, sensor_type):
# longName = 'Sibratsgfall test' # string
# Offering values
off_name = '\"' + str(offering.name) + '\"' # Offering name, double quoted
gml_id = '\"' + str(procedure.id) + '\"' # Offering name, double quoted
offering_name = offering.name
offering_label = offering.label
# offID = offering.fullId # URL format of full id
@ -165,16 +135,17 @@ def insert_sensor(offering, procedure, foi, sensor_type):
"service": "SOS",
"version": "2.0.0",
"procedureDescriptionFormat": "http://www.opengis.net/sensorml/2.0",
"procedureDescription": f'<sml:PhysicalSystem gml:id={off_name} xmlns:swes=\"http://www.opengis.net/swes/2.0\" xmlns:sos=\"http://www.opengis.net/sos/2.0\" xmlns:swe=\"http://www.opengis.net/swe/2.0\" xmlns:sml=\"http://www.opengis.net/sensorml/2.0\" xmlns:gml=\"http://www.opengis.net/gml/3.2\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:gco=\"http://www.isotc211.org/2005/gco\" xmlns:gmd=\"http://www.isotc211.org/2005/gmd\"><gml:identifier codeSpace=\"uniqueID\">{procedure_identifier}</gml:identifier><sml:identification><sml:IdentifierList><sml:identifier><sml:Term definition=\"urn:ogc:def:identifier:OGC:1.0:shortName\"><sml:label>shortName</sml:label><sml:value>{procedure_name}</sml:value></sml:Term></sml:identifier></sml:IdentifierList></sml:identification><sml:capabilities name=\"offerings\"><sml:CapabilityList><sml:capability name=\"offeringID\"><swe:Text definition=\"urn:ogc:def:identifier:OGC:offeringID\"><swe:label>{offering_label}</swe:label><swe:value>{offering_name}</swe:value></swe:Text></sml:capability></sml:CapabilityList></sml:capabilities><sml:featuresOfInterest><sml:FeatureList definition=\"http://www.opengis.net/def/featureOfInterest/identifier\"><swe:label>featuresOfInterest</swe:label><sml:feature><sams:SF_SpatialSamplingFeature xmlns:sams=\"http://www.opengis.net/samplingSpatial/2.0\" gml:id=\"ssf_b3a826dd44012201b01323232323041f7a92e0cc47260eb9888f6a4e9f747\"><gml:identifier codeSpace=\"http://www.opengis.net/def/nil/OGC/0/unknown\">{feature_id}</gml:identifier><gml:name codeSpace=\"http://www.opengis.net/def/nil/OGC/0/unknown\">{feature_name}</gml:name><sf:type xmlns:sf=\"http://www.opengis.net/sampling/2.0\" xlink:href=\"http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint\"/><sf:sampledFeature xmlns:sf=\"http://www.opengis.net/sampling/2.0\" xlink:href=\"http://www.opengis.net/def/nil/OGC/0/unknown\"/><sams:shape><ns:Point xmlns:ns=\"http://www.opengis.net/gml/3.2\" ns:id=\"Point_ssf_b3a826dd44012201b013c90c51da28c041f7a92e0cc47260eb9888f6a4e9f747\"><ns:pos srsName=\"http://www.opengis.net/def/crs/EPSG/0/4326\">{coordinates}</ns:pos></ns:Point></sams:shape></sams:SF_SpatialSamplingFeature></sml:feature></sml:FeatureList></sml:featuresOfInterest><sml:outputs><sml:OutputList><sml:output name=\"Slope\"><swe:Quantity definition=\"Slope\"><swe:label>Slope</swe:label><swe:uom code=\"deg\"/></swe:Quantity></sml:output></sml:OutputList></sml:outputs><sml:position><swe:Vector referenceFrame=\"urn:ogc:def:crs:EPSG::4326\"><swe:coordinate name=\"easting\"><swe:Quantity axisID=\"x\"><swe:uom code=\"degree\" /><swe:value>{cord_x}</swe:value></swe:Quantity></swe:coordinate><swe:coordinate name=\"northing\"><swe:Quantity axisID=\"y\"><swe:uom code=\"degree\" /><swe:value>{cord_y}</swe:value></swe:Quantity></swe:coordinate><swe:coordinate name=\"altitude\"><swe:Quantity axisID=\"z\"><swe:uom code=\"m\" /><swe:value>{height}</swe:value></swe:Quantity></swe:coordinate></swe:Vector></sml:position></sml:PhysicalSystem>',
"procedureDescription": f'<sml:PhysicalSystem gml:id={gml_id} xmlns:swes=\"http://www.opengis.net/swes/2.0\" xmlns:sos=\"http://www.opengis.net/sos/2.0\" xmlns:swe=\"http://www.opengis.net/swe/2.0\" xmlns:sml=\"http://www.opengis.net/sensorml/2.0\" xmlns:gml=\"http://www.opengis.net/gml/3.2\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:gco=\"http://www.isotc211.org/2005/gco\" xmlns:gmd=\"http://www.isotc211.org/2005/gmd\"><gml:identifier codeSpace=\"uniqueID\">{procedure_identifier}</gml:identifier><sml:identification><sml:IdentifierList><sml:identifier><sml:Term definition=\"urn:ogc:def:identifier:OGC:1.0:longName\"><sml:label>longName</sml:label><sml:value>{procedure_name}</sml:value></sml:Term></sml:identifier><sml:identifier><sml:Term definition=\"urn:ogc:def:identifier:OGC:1.0:shortName\"><sml:label>shortName</sml:label><sml:value>{procedure_name}</sml:value></sml:Term></sml:identifier></sml:IdentifierList></sml:identification><sml:capabilities name=\"offerings\"><sml:CapabilityList><sml:capability name=\"offeringID\"><swe:Text definition=\"urn:ogc:def:identifier:OGC:offeringID\"><swe:label>{offering_label}</swe:label><swe:value>{offering_name}</swe:value></swe:Text></sml:capability></sml:CapabilityList></sml:capabilities><sml:capabilities name=\"metadata\"><sml:CapabilityList><!-- status indicates, whether sensor is insitu (true) or remote (false) --><sml:capability name=\"insitu\"><swe:Boolean definition=\"insitu\"><swe:value>true</swe:value></swe:Boolean></sml:capability><!-- status indicates, whether sensor is mobile (true) or fixed/stationary (false) --><sml:capability name=\"mobile\"><swe:Boolean definition=\"mobile\"><swe:value>false</swe:value></swe:Boolean></sml:capability></sml:CapabilityList></sml:capabilities><sml:featuresOfInterest><sml:FeatureList definition=\"http://www.opengis.net/def/featureOfInterest/identifier\"><swe:label>featuresOfInterest</swe:label><sml:feature><sams:SF_SpatialSamplingFeature xmlns:sams=\"http://www.opengis.net/samplingSpatial/2.0\" gml:id=\"ssf_b3a826dd44012201b01323232323041f7a92e0cc47260eb9888f6a4e9f747\"><gml:identifier codeSpace=\"http://www.opengis.net/def/nil/OGC/0/unknown\">{feature_id}</gml:identifier><gml:name codeSpace=\"http://www.opengis.net/def/nil/OGC/0/unknown\">{feature_name}</gml:name><sf:type xmlns:sf=\"http://www.opengis.net/sampling/2.0\" xlink:href=\"http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint\"/><sf:sampledFeature xmlns:sf=\"http://www.opengis.net/sampling/2.0\" xlink:href=\"http://www.opengis.net/def/nil/OGC/0/unknown\"/><sams:shape><ns:Point xmlns:ns=\"http://www.opengis.net/gml/3.2\" ns:id=\"Point_ssf_b3a826dd44012201b013c90c51da28c041f7a92e0cc47260eb9888f6a4e9f747\"><ns:pos srsName=\"http://www.opengis.net/def/crs/EPSG/0/4326\">{coordinates}</ns:pos></ns:Point></sams:shape></sams:SF_SpatialSamplingFeature></sml:feature></sml:FeatureList></sml:featuresOfInterest><sml:outputs><sml:OutputList><sml:output name=\"Elevation\"><swe:Quantity definition=\"Elevation\"><swe:label>Elevation</swe:label><swe:uom code=\"m\"/></swe:Quantity></sml:output></sml:OutputList></sml:outputs><sml:position><swe:Vector referenceFrame=\"urn:ogc:def:crs:EPSG::4326\"><swe:coordinate name=\"easting\"><swe:Quantity axisID=\"x\"><swe:uom code=\"degree\"/><swe:value>{cord_x}</swe:value></swe:Quantity></swe:coordinate><swe:coordinate name=\"northing\"><swe:Quantity axisID=\"y\"><swe:uom code=\"degree\"/><swe:value>{cord_y}</swe:value></swe:Quantity></swe:coordinate><swe:coordinate name=\"altitude\"><swe:Quantity axisID=\"z\"><swe:uom code=\"m\"/><swe:value>{height}</swe:value></swe:Quantity></swe:coordinate></swe:Vector></sml:position></sml:PhysicalSystem>',
"observableProperty": [
"Slope",
"Elevation",
# "Roll",
# "InSystemTemperature"
],
"observationType": [
"http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"
],
"featureOfInterestType": "http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint"
"featureOfInterestType":
"http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint"
}
return body

View File

@ -6,10 +6,11 @@ Python version: 3.7
import json
import os
import uuid
from datetime import datetime
from sqlalchemy.orm import session
from sqlalchemy import asc, desc, func
from dotenv import load_dotenv, find_dotenv
import requests
from datetime import datetime
from db.models import (
Observation,
create_pg_session,
@ -26,9 +27,9 @@ def main():
pg_session: session = create_pg_session()
platform_sta_identifier = "pechgraben_piezometer"
# sensor = "bohrloch1"
# sensor_list = os.environ.get('GLASFASER_GSCHLIEFGRABEN_SENSORS', [])
sensor_list = json.loads(os.environ['GLASFASER_GSCHLIEFGRABEN_SENSORS'])
# sensor_list = os.environ.get('PIEZOMETER_GSCHLIEFGRABEN_SENSORS', [])
sensor_list = json.loads(os.environ['PIEZOMETER_GSCHLIEFGRABEN_SENSORS'])
url = 'https://jaa5ixl2y0.execute-api.ap-southeast-2.amazonaws.com/v1/data'
params = {}
headers = {'content-type': 'application/json'}
@ -44,7 +45,7 @@ def main():
Phenomenon.sta_identifier == "Elevation").first()
if not elevation_dataset:
print("Sensor " + sensor + " ist noch nicht angelegt!")
exit()
continue
if not elevation_dataset.is_published:
elevation_dataset.is_published = 1
elevation_dataset.is_hidden = 0
@ -55,7 +56,21 @@ def main():
platform_exists: bool = pg_session.query(Platform.id).filter_by(
sta_identifier=platform_sta_identifier).scalar() is not None
if platform_exists:
# if platform_exists:
# sensor_platform = pg_session.query(Platform.id) \
# .filter(Platform.sta_identifier == platform_sta_identifier) \
# .first()
# elevation_dataset.fk_platform_id = sensor_platform.id
if not platform_exists:
sensor_platform = Platform()
max_id = pg_session.query(func.max(Platform.id)).scalar()
# sensor_platform.id = max_id + 1
sensor_platform.sta_identifier = platform_sta_identifier.lower()
sensor_platform.identifier = platform_sta_identifier.lower()
sensor_platform.name = platform_sta_identifier.lower()
elevation_dataset.platform = sensor_platform
else:
sensor_platform = pg_session.query(Platform.id) \
.filter(Platform.sta_identifier == platform_sta_identifier) \
.first()
@ -66,16 +81,39 @@ def main():
).scalar() is not None
if format_exists:
sensor_format = pg_session.query(Format.id) \
.filter(Format.definition == "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement") \
.filter(Format.definition ==
"http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement") \
.first()
elevation_dataset.fk_format_id = sensor_format.id
if sensor in data:
create_observation(elevation_dataset, sensor, data, pg_session)
pg_session.commit()
first_elevation_observation = pg_session.query(Observation) \
.filter(Observation.fk_dataset_id == elevation_dataset.id) \
.order_by(asc('sampling_time_start')) \
.first()
if first_elevation_observation is not None:
elevation_dataset.first_time = first_elevation_observation.sampling_time_start
elevation_dataset.first_value = first_elevation_observation.value_quantity
elevation_dataset.fk_first_observation_id = first_elevation_observation.id
last_elevation_observation = pg_session.query(Observation) \
.filter(Observation.fk_dataset_id == elevation_dataset.id) \
.order_by(desc('sampling_time_start')) \
.first()
if last_elevation_observation is not None:
elevation_dataset.last_time = last_elevation_observation.sampling_time_start
elevation_dataset.last_value = last_elevation_observation.value_quantity
elevation_dataset.fk_last_observation_id = last_elevation_observation.id
pg_session.commit()
pg_session.close()
def create_observation(elevation_dataset: Dataset, sensor_key: str, data: json, db_session: session):
def create_observation(elevation_dataset: Dataset,
sensor_key: str,
data: json,
db_session: session):
''' create observation in db'''
print("Sesnor key exist in JSON data")
sensor_object = data[sensor_key]
@ -86,7 +124,8 @@ def create_observation(elevation_dataset: Dataset, sensor_key: str, data: json,
existing_observation: bool = (
db_session.query(Observation)
.filter(Observation.result_time == date_obj, Observation.fk_dataset_id == elevation_dataset.id)
.filter(Observation.result_time ==
date_obj, Observation.fk_dataset_id == elevation_dataset.id)
.one_or_none()
)
# Can we insert this observation?
@ -100,6 +139,7 @@ def create_observation(elevation_dataset: Dataset, sensor_key: str, data: json,
new_observation.sampling_time_end = new_observation.result_time
new_observation.value_type = "quantity"
new_observation.value_quantity = abstich
new_observation.fk_dataset_id = elevation_dataset.id
db_session.add(new_observation)
@ -135,6 +175,6 @@ def test():
if __name__ == "__main__":
load_dotenv(find_dotenv())
sensor_list1 = os.environ.get('GLASFASER_GSCHLIEFGRABEN_SENSORS', [])
sensor_list1 = os.environ.get('PIEZOMETER_GSCHLIEFGRABEN_SENSORS', [])
print(f'sensors: {sensor_list1} .')
main()

View File

@ -71,10 +71,13 @@ class SensorType:
("Battery level", "m"), ("Temperature", "m"), ("Relative humidity", "m")]}
elif type_ == "inclinometer": # INCLINOMETER
self.pattern = {"name": "inclinometer", "type": 'fixed', "attributes": [
("Slope", "m"), ("Roll", "m")]}
("Slope", "deg"), ("Roll", "deg")]}
elif type_ == "camera": # INCLINOMETER
self.pattern = {"name": "camera", "type": 'fixed', "attributes": [
("Image", "xo")]}
elif type_ == "piezometer": # PIEZOMETER
self.pattern = {"name": "piezometer", "type": 'fixed', "attributes": [
("Elevation", "m")]}
elif type_ == "noise": # NOISE
self.pattern = {"name": "noise", "type": 'fixed', "attributes": [
("Battery level", "m"), ("Noise", "m")]}

View File

@ -73,7 +73,7 @@ def main():
# headers = {'Content-Type': 'application/soap+xml'} # set what your server accepts
# xml = get_xml(offering, procedure, foi, sensor_type)
# xml = get_xml(offering, procedure, foi, sensor_type)
# request = requests.post(sos_url, data = xml, headers=headers)
# print(request.text)
@ -229,7 +229,8 @@ def insert_sensor(offering, procedure, foi, sensor_type):
"observationType": [
"http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_TextObservation"
],
"featureOfInterestType": "http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint"
"featureOfInterestType":
"http://www.opengis.net/def/samplingFeatureType/OGC-OM/2.0/SF_SamplingPoint"
}
return body