diff --git a/gschliefgraben_glasfaser/main.py b/gschliefgraben_glasfaser/main.py index 7bea507..09d852b 100644 --- a/gschliefgraben_glasfaser/main.py +++ b/gschliefgraben_glasfaser/main.py @@ -6,16 +6,19 @@ Python version: 3.7 ''' import os +from tokenize import String +import uuid # import sys, inspect # currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # parentdir = os.path.dirname(currentdir) # sys.path.insert(0, parentdir) # import requests from sqlalchemy.orm import session - -from gschliefgraben_glasfaser.models import ObservationSchema, Person, PersonSchema, Observation +from sqlalchemy import func +from gschliefgraben_glasfaser.models import ObservationSchema, Person, PersonSchema, Observation, create_pg_session from gschliefgraben_glasfaser.my_api import MyApi -from db.pg_models import create_pg_session +from datetime import datetime, date, timedelta +# from db.pg_models import create_pg_session #from models import Person, PersonSchema # response = requests.get('https://api.com/') # print(response) # shows the response's HTTP status code @@ -47,8 +50,8 @@ def main(): # print(load_data) # request ortmann api - # token = 'eyJraWQiOiJlakFmX1MwMTBMU3doS0Zod05wZDQtQkZPYTM4cDRYRE1zU1hFa0lrRlhFIiwiYWxnIjoiUlMyNTYifQ.eyJ2ZXIiOjEsImp0aSI6IkFULkZRUHNCOWh5Snd6eEM5d3ZWelRvaTNpZVlMWlJiT3U4YzFCbWJWRGM1SFkiLCJpc3MiOiJodHRwczovL2Rldi01MjUwMDA2Lm9rdGEuY29tL29hdXRoMi9kZWZhdWx0IiwiYXVkIjoiYXBpOi8vZGVmYXVsdCIsImlhdCI6MTY0NTc4Mjg0NSwiZXhwIjoxNjQ1Nzg2NDQ1LCJjaWQiOiIwb2EyOWhzdGZ3RnFya1BrUDVkNyIsInNjcCI6WyJnc2NobGllZmdyYWJlbiJdLCJzdWIiOiIwb2EyOWhzdGZ3RnFya1BrUDVkNyJ9.c-pTs-3VJMnFO2SOqxOvsABAloprUmOjk6SO9J71NrgLj7claKZOMLZxRyUeSBLWCJFFNI3A6xMd4twEexjJdUR8UEM4U50srxr2p_enaMm1_jZTSt_76u6H05kwV-A2AOQPkx-Fxxaj_PDjT7w43Zlg6SUEoT11uGKR6KtxVYbclGtWgOR7wvH4NZav-P_EDjHwHxbk2kQSf7tBU1JbWl74Xt58gzv1t8VNtLYLICabRsuTNQUNiO7Y1rtUEav4ugf7WZMIY1cP_4rCupZrAFbxrnyprAuXA2x01Z9hbFmiaK0QDlrwHcCHL_1fKvj9uIbO5JeI1x81X6g7eAxQdA' - # response = requests.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=("inclino1_14")', + # response = + # requests.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=("inclino1_14")', # headers={ # 'Authorization': 'Bearer' + token, # 'cache-control': 'no-cache', @@ -57,16 +60,116 @@ def main(): # }, # data='grant_type=client_credentials&scope=gschliefgraben') # print(response) + + # The size of each step in days + + # consider the start date as 2021-february 1 st + start_date = date(2021, 2, 28) + # consider the end date as 2021-march 1 st + end_date = date(2022, 3, 1) + + # delta time + delta = timedelta(days=1) token_api = os.environ.get("TOKEN_API") test_api = MyApi(token_api) - data = test_api.getSensorData("inclino1_14") - observation_array = (data['FeatureCollection']['Features'][0]['geometry']['properties'][0]) - print(observation_array) - # create(dump_data) - # # deserialize to db model - observation_schema = ObservationSchema(many=True) - observations: Observation = observation_schema.load(observation_array) - print(observations) + + # iterate over range of dates + while start_date <= end_date: + # print(start_date, end="\n") + query_date = start_date.strftime('%Y-%m-%d') + create_db_observations(query_date, test_api, pg_session) + start_date += delta + + # for i in rrule(DAILY , dtstart=start_date,until=end_date): + # print(i.strftime('%Y%b%d'),sep='\n') + + + # query_date = "2022-02-28" + # create_db_observations(query_date, test_api, pg_session) + # query_date_obj = datetime.strptime(query_date, "%Y-%m-%d") + # data = test_api.getSensorData("inclino1_14", query_date) + # observation_array = (data['FeatureCollection'] + # ['Features'][0]['geometry']['properties'][0]) + # print(observation_array) + + + + # max_id = pg_session.query(func.max(Observation.id)).scalar() + # if max_id is None: + # max_id = -1 + # # pg_session.bulk_save_objects(observations) + # for observation_json in observation_array: + # ob_date_time = observation_json.get('DateTime') + # datetime_obj = datetime.strptime(ob_date_time, "%Y-%m-%dT%H:%M:%S.%fZ") + # if datetime_obj.date() != query_date_obj.date(): + # continue + # max_id = max_id + 1 + # create_observation(observation_json, pg_session, max_id) + + # pg_session.commit() + +def create_db_observations(query_date, test_api, pg_session): + ''' to do ''' + query_date_obj = datetime.strptime(query_date, "%Y-%m-%d") + data = test_api.getSensorData("inclino1_14", query_date) + observation_array = (data['FeatureCollection'] + ['Features'][0]['geometry']['properties'][0]) + # print(observation_array) + + max_id = pg_session.query(func.max(Observation.id)).scalar() + if max_id is None: + max_id = -1 + # pg_session.bulk_save_objects(observations) + for observation_json in observation_array: + ob_date_time = observation_json.get('DateTime') + datetime_obj = datetime.strptime(ob_date_time, "%Y-%m-%dT%H:%M:%S.%fZ") + if datetime_obj.date() != query_date_obj.date(): + continue + ob_value = observation_json.get('Value') + if ob_value is None: + continue + max_id = max_id + 1 + create_observation(observation_json, pg_session, max_id) + pg_session.commit() + print("observations for date " +query_date+ "succesfully imported \n") + + +def create_observation(observation_json: ObservationSchema, db_session, max_id): + """ + This function creates a new observation in the people structure + based on the passed-in observation data + :param observation: person to create in people structure + :return: 201 on success, observation on person exists + """ + + ob_id = observation_json.get('id') + # db_session = create_pg_session() + + existing_observation: bool = ( + db_session.query(Observation) + .filter(Observation.id == ob_id) + .one_or_none() + ) + + # Can we insert this observation? + if existing_observation is None: + # Create a person instance using the schema and the passed in person + schema = ObservationSchema() + # deserialize to object + new_observation: Observation = schema.load(observation_json) + new_observation.id = max_id + 1 + new_observation.sta_identifier = str(uuid.uuid4()) + + # Add the person to the database + db_session.add(new_observation) + # db_session.commit() + + # Serialize and return the newly created person in the response + data = schema.dump(new_observation) + return data, 201 + # Otherwise, nope, person exists already + else: + print(409, f'Observation {ob_id} exists already') def create(person_json: PersonSchema): diff --git a/gschliefgraben_glasfaser/models.py b/gschliefgraben_glasfaser/models.py index 419b564..1405794 100644 --- a/gschliefgraben_glasfaser/models.py +++ b/gschliefgraben_glasfaser/models.py @@ -10,25 +10,19 @@ from datetime import datetime import os from sqlalchemy import (Column, Integer, - String, DateTime, ForeignKey, Numeric, SmallInteger, create_engine) + String, DateTime, ForeignKey, Numeric, SmallInteger, create_engine, func) from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import session, relationship +from sqlalchemy.orm import session, relationship, sessionmaker #from marshmallow import Schema from marshmallow_sqlalchemy import SQLAlchemySchema, SQLAlchemyAutoSchema from marshmallow import fields # from db.pg_models import create_pg_session -from sqlalchemy import func, desc, asc -import sqlalchemy.orm.session -from sqlalchemy.orm import sessionmaker, relationship +# import sqlalchemy.orm.session Base = declarative_base() -def new_id_factory(): - pg_session = create_pg_session() - _MYTABLE_ID_ = pg_session.query(func.max(Observation.id)).scalar() - _MYTABLE_ID_ += 1 - return _MYTABLE_ID_ -def create_pg_session() -> sqlalchemy.orm.sessionmaker: + +def create_pg_session() -> sessionmaker: """ create postgres db session """ dbschema = '' db_user = os.environ.get("POSTGIS_DBUSER") @@ -39,10 +33,11 @@ def create_pg_session() -> sqlalchemy.orm.sessionmaker: connect_args={'options': '-csearch_path={}'.format(dbschema)}, isolation_level="READ UNCOMMITTED") session_maker = sessionmaker(bind=engine) - session = session_maker() + _session = session_maker() # Base.metadata.create_all(engine) - return session + return _session + class Dataset(Base): """ dataset class """ @@ -75,29 +70,47 @@ class Dataset(Base): # first_observation = relationship("Observation", foreign_keys=[ # fk_first_observation_id]) - observations = relationship( - 'Observation', back_populates='dataset', lazy=True) - + # observations = relationship( + # 'Observation', back_populates='dataset', lazy=True) + +def new_id_factory(): + ''' test ''' + dbschema = '' + db_user = os.environ.get("POSTGIS_DBUSER") + db_password = os.environ.get("POSTGIS_DBPASSWORD") + db_url = os.environ.get("POSTGIS_DBURL") + engine = create_engine( + "postgresql+psycopg2://" + db_user + ":" + db_password + "@" + db_url, + connect_args={'options': '-csearch_path={}'.format(dbschema)}, + isolation_level="READ UNCOMMITTED") + result = engine.execute('SELECT MAX(observation_id) FROM gba.observation') + mytable_max_id = result.first().max + if mytable_max_id is None: + mytable_max_id = 0 + mytable_max_id += 1 + return mytable_max_id + + class Observation(Base): """ observation class """ __tablename__ = 'observation' __table_args__ = {"schema": "gba"} - id = Column('observation_id', Integer, - primary_key=True) + id = Column('observation_id', Integer, primary_key=True) name = Column('name', String) - value_type = Column('value_type', String) + value_type = Column('value_type', String, default="quantity") # pitch = Column('PITCH', String) # roll = Column('ROLL', String) sampling_time_start = Column('sampling_time_start', DateTime) sampling_time_end = Column('sampling_time_end', DateTime) result_time = Column('result_time', DateTime) sta_identifier = Column('sta_identifier', String) + value_identifier = Column('value_identifier', String) value_quantity = Column('value_quantity', Numeric(20, 10), nullable=False) - fk_dataset_id = Column(Integer, ForeignKey( - 'gba.dataset.dataset_id'), nullable=False) - dataset = relationship("Dataset", back_populates="observations") + # fk_dataset_id = Column(Integer, ForeignKey( + # 'gba.dataset.dataset_id'), nullable=False) + # dataset = relationship("Dataset", back_populates="observations") class ObservationSchema(SQLAlchemySchema): @@ -106,7 +119,7 @@ class ObservationSchema(SQLAlchemySchema): # value_quantity = fields.Integer(attribute='Value') # id = fields.Integer(attribute='id') Value = fields.Integer(attribute='value_quantity') - id = fields.Integer(attribute='sta_identifier') + id = fields.Integer(attribute='value_identifier') # sta_identifier= fields.String(default=uuid.uuid4()), class Meta: @@ -134,7 +147,7 @@ class Person(Base): self.login, self.lname) -class PersonSchema(SQLAlchemyAutoSchema): +class PersonSchema(SQLAlchemyAutoSchema): """ Platform class """ class Meta: """ Platform class """ @@ -144,12 +157,12 @@ class PersonSchema(SQLAlchemyAutoSchema): #pg_session: session = create_pg_session() sqla_session: session = create_pg_session() + def create_db(): - # db_url = 'sqlite:///db.sqlite' - # engine = create_engine(db_url, echo = True ) + # db_url = 'sqlite:///db.sqlite' + # engine = create_engine(db_url, echo = True ) # Base.metadata.drop_all(bind=engine) # Base.metadata.create_all(engine) - """ create postgres db session """ dbschema = '' db_user = os.environ.get("POSTGIS_DBUSER") @@ -158,11 +171,12 @@ def create_db(): engine = create_engine( "postgresql+psycopg2://" + db_user + ":" + db_password + "@" + db_url, connect_args={'options': '-csearch_path={}'.format(dbschema)}, - isolation_level="READ UNCOMMITTED", echo = True) + isolation_level="READ UNCOMMITTED", echo=True) # session_maker = sessionmaker(bind=engine) # session = session_maker() Base.metadata.drop_all(bind=engine) Base.metadata.create_all(engine) - + + if __name__ == "__main__": - create_db() \ No newline at end of file + create_db() diff --git a/gschliefgraben_glasfaser/my_api.py b/gschliefgraben_glasfaser/my_api.py index f625ced..f6fa8cd 100644 --- a/gschliefgraben_glasfaser/my_api.py +++ b/gschliefgraben_glasfaser/my_api.py @@ -65,10 +65,10 @@ class MyApi(): # self.access_token = res.json()['access_token'] # else: # # Token expired -> re-authenticate - def getSensorData(self, sensor: string): + def getSensorData(self, sensor: string, date): ''' request observations''' try: - request = self.session.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=(\''+sensor+ '\')&start=2022-02-28&end=2022-02-28', + request = self.session.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=(\''+sensor+ '\')&start='+date+'&end='+date, headers={ 'cache-control': 'no-cache', 'Content-Type': 'application/x-www-form-urlencoded', diff --git a/insert_sensor/execute.py b/insert_sensor/execute.py index 2f71235..695cee3 100644 --- a/insert_sensor/execute.py +++ b/insert_sensor/execute.py @@ -38,14 +38,25 @@ def main(): sos_url = 'https://geomon.geologie.ac.at/52n-sos-webapp/service' ######################## Sibratsgfall + # offering = Offering( + # "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/", + # "sibratsgfall_3", + # "Inklinometer 3, Sibratsgfaell Sensor" + # ) + # procedure = Procedure( "sibratsgfall_3","sibratsgfall-3") + # foi = FoI("degree", "m", (47.4279288, 10.0360888, 0.0), + # "sibratsgfall", "Sibratsgfall Beobachtung der Bodenbewegungen Test") + + ####################### Gschliefgraben Glasfaser offering = Offering( "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/", - "sibratsgfall_3", - "Inklinometer 3, Sibratsgfaell Sensor" + "inclino1_14", + "Inklinometer inclino1_14, Gschliefgraben Glasfaser" ) - procedure = Procedure( "sibratsgfall_3","sibratsgfall-3") - foi = FoI("degree", "m", (47.4279288, 10.0360888, 0.0), - "sibratsgfall", "Sibratsgfall Beobachtung der Bodenbewegungen Test") + procedure = Procedure( "inclino1_14","inclino1_14") + + foi = FoI("degree", "m", (47.910849, 13.774966, 0.0), + "FBGuard23", "Glasfaser Untersuchungen am Gschliefgraben (Gmunden)") ######################## Gschliefgraben # offering = Offering( @@ -77,6 +88,13 @@ def main(): request = requests.post(sos_url, headers=headers, json=post_data) print(request.text) + # { + # "request" : "InsertSensor", + # "version" : "2.0.0", + # "service" : "SOS", + # "assignedProcedure" : "inclino1_14", + # "assignedOffering" : "inclino1_14" + # } if __name__ == '__main__':