- check if observation already exits in Db, then insert

gschliefgraben_glasfaser/main.py

@@ -6,16 +6,19 @@ Python version: 3.7
 '''
 
 import os
+from tokenize import String
+import uuid
 # import sys, inspect
 # currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
 # parentdir = os.path.dirname(currentdir)
 # sys.path.insert(0, parentdir)
 # import requests
 from sqlalchemy.orm import session
-
+from sqlalchemy import func
-from gschliefgraben_glasfaser.models import ObservationSchema, Person, PersonSchema, Observation
+from gschliefgraben_glasfaser.models import ObservationSchema, Person, PersonSchema, Observation, create_pg_session
 from gschliefgraben_glasfaser.my_api import MyApi
-from db.pg_models import create_pg_session
+from datetime import datetime, date, timedelta
+# from db.pg_models import create_pg_session
 #from models import Person, PersonSchema
 # response = requests.get('https://api.com/')
 # print(response)  # shows the response's HTTP status code
@@ -47,8 +50,8 @@ def main():
     # print(load_data)
 
     # request ortmann api
-    # token = 'eyJraWQiOiJlakFmX1MwMTBMU3doS0Zod05wZDQtQkZPYTM4cDRYRE1zU1hFa0lrRlhFIiwiYWxnIjoiUlMyNTYifQ.eyJ2ZXIiOjEsImp0aSI6IkFULkZRUHNCOWh5Snd6eEM5d3ZWelRvaTNpZVlMWlJiT3U4YzFCbWJWRGM1SFkiLCJpc3MiOiJodHRwczovL2Rldi01MjUwMDA2Lm9rdGEuY29tL29hdXRoMi9kZWZhdWx0IiwiYXVkIjoiYXBpOi8vZGVmYXVsdCIsImlhdCI6MTY0NTc4Mjg0NSwiZXhwIjoxNjQ1Nzg2NDQ1LCJjaWQiOiIwb2EyOWhzdGZ3RnFya1BrUDVkNyIsInNjcCI6WyJnc2NobGllZmdyYWJlbiJdLCJzdWIiOiIwb2EyOWhzdGZ3RnFya1BrUDVkNyJ9.c-pTs-3VJMnFO2SOqxOvsABAloprUmOjk6SO9J71NrgLj7claKZOMLZxRyUeSBLWCJFFNI3A6xMd4twEexjJdUR8UEM4U50srxr2p_enaMm1_jZTSt_76u6H05kwV-A2AOQPkx-Fxxaj_PDjT7w43Zlg6SUEoT11uGKR6KtxVYbclGtWgOR7wvH4NZav-P_EDjHwHxbk2kQSf7tBU1JbWl74Xt58gzv1t8VNtLYLICabRsuTNQUNiO7Y1rtUEav4ugf7WZMIY1cP_4rCupZrAFbxrnyprAuXA2x01Z9hbFmiaK0QDlrwHcCHL_1fKvj9uIbO5JeI1x81X6g7eAxQdA'
+    # response =
-    # response = requests.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=("inclino1_14")',
+    # requests.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=("inclino1_14")',
     #                         headers={
     #                             'Authorization': 'Bearer' + token,
     #                             'cache-control': 'no-cache',
@@ -57,16 +60,116 @@ def main():
     #                         },
     #                         data='grant_type=client_credentials&scope=gschliefgraben')
     # print(response)
+
+    # The size of each step in days
+
+    # consider the start date as 2021-february 1 st
+    start_date = date(2021, 2, 28)
+    # consider the end date as 2021-march 1 st
+    end_date = date(2022, 3, 1)
+
+    # delta time
+    delta = timedelta(days=1)
     token_api = os.environ.get("TOKEN_API")
     test_api = MyApi(token_api)
-    data = test_api.getSensorData("inclino1_14")
+
-    observation_array = (data['FeatureCollection']['Features'][0]['geometry']['properties'][0])
+    # iterate over range of dates
-    print(observation_array)
+    while start_date <= end_date:
-    # create(dump_data)
+        # print(start_date, end="\n")
-    # # deserialize to db model
+        query_date = start_date.strftime('%Y-%m-%d')
-    observation_schema = ObservationSchema(many=True)
+        create_db_observations(query_date, test_api, pg_session)
-    observations: Observation = observation_schema.load(observation_array)
+        start_date += delta
-    print(observations)
+
+    # for i in rrule(DAILY , dtstart=start_date,until=end_date):
+    #     print(i.strftime('%Y%b%d'),sep='\n')
+
+
+    # query_date = "2022-02-28"
+    # create_db_observations(query_date, test_api, pg_session)
+    # query_date_obj = datetime.strptime(query_date, "%Y-%m-%d")
+    # data = test_api.getSensorData("inclino1_14", query_date)
+    # observation_array = (data['FeatureCollection']
+    #                      ['Features'][0]['geometry']['properties'][0])
+    # print(observation_array)
+
+
+
+    # max_id = pg_session.query(func.max(Observation.id)).scalar()
+    # if max_id is None:
+    #     max_id = -1
+    # # pg_session.bulk_save_objects(observations)
+    # for observation_json in observation_array:
+    #     ob_date_time = observation_json.get('DateTime')      
+    #     datetime_obj = datetime.strptime(ob_date_time, "%Y-%m-%dT%H:%M:%S.%fZ")
+    #     if datetime_obj.date() != query_date_obj.date():
+    #         continue
+    #     max_id = max_id + 1
+    #     create_observation(observation_json, pg_session, max_id)
+
+    # pg_session.commit()
+
+def create_db_observations(query_date, test_api, pg_session):
+    ''' to do '''
+    query_date_obj = datetime.strptime(query_date, "%Y-%m-%d")
+    data = test_api.getSensorData("inclino1_14", query_date)
+    observation_array = (data['FeatureCollection']
+                         ['Features'][0]['geometry']['properties'][0])
+    # print(observation_array)
+
+    max_id = pg_session.query(func.max(Observation.id)).scalar()
+    if max_id is None:
+        max_id = -1
+    # pg_session.bulk_save_objects(observations)
+    for observation_json in observation_array:
+        ob_date_time = observation_json.get('DateTime')      
+        datetime_obj = datetime.strptime(ob_date_time, "%Y-%m-%dT%H:%M:%S.%fZ")
+        if datetime_obj.date() != query_date_obj.date():
+            continue
+        ob_value = observation_json.get('Value') 
+        if ob_value is None:
+            continue
+        max_id = max_id + 1
+        create_observation(observation_json, pg_session, max_id)
+    pg_session.commit()
+    print("observations for date " +query_date+  "succesfully imported \n")
+
+
+def create_observation(observation_json: ObservationSchema, db_session, max_id):
+    """
+    This function creates a new observation in the people structure
+    based on the passed-in observation data
+    :param observation:  person to create in people structure
+    :return:        201 on success, observation on person exists
+    """
+
+    ob_id = observation_json.get('id')
+    # db_session = create_pg_session()
+
+    existing_observation: bool = (
+        db_session.query(Observation)
+        .filter(Observation.id == ob_id)
+        .one_or_none()
+    )
+
+    # Can we insert this observation?
+    if existing_observation is None:
+        # Create a person instance using the schema and the passed in person
+        schema = ObservationSchema()
+        # deserialize to object
+        new_observation: Observation = schema.load(observation_json)
+        new_observation.id = max_id + 1
+        new_observation.sta_identifier = str(uuid.uuid4())
+
+        # Add the person to the database
+        db_session.add(new_observation)
+        # db_session.commit()
+
+        # Serialize and return the newly created person in the response
+        data = schema.dump(new_observation)
+        return data, 201
+    # Otherwise, nope, person exists already
+    else:
+        print(409, f'Observation {ob_id} exists already')
 
 
 def create(person_json: PersonSchema):

gschliefgraben_glasfaser/models.py

@@ -10,25 +10,19 @@ from datetime import datetime
 
 import os
 from sqlalchemy import (Column, Integer,
-                        String, DateTime, ForeignKey, Numeric, SmallInteger, create_engine)
+                        String, DateTime, ForeignKey, Numeric, SmallInteger, create_engine, func)
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import session, relationship
+from sqlalchemy.orm import session, relationship, sessionmaker
 #from marshmallow import Schema
 from marshmallow_sqlalchemy import SQLAlchemySchema, SQLAlchemyAutoSchema
 from marshmallow import fields
 # from db.pg_models import create_pg_session
-from sqlalchemy import func, desc, asc
+# import sqlalchemy.orm.session
-import sqlalchemy.orm.session
-from sqlalchemy.orm import sessionmaker, relationship
 
 Base = declarative_base()
-def new_id_factory():
-    pg_session = create_pg_session()
-    _MYTABLE_ID_ = pg_session.query(func.max(Observation.id)).scalar()
-    _MYTABLE_ID_ += 1
-    return _MYTABLE_ID_
 
-def create_pg_session() -> sqlalchemy.orm.sessionmaker:
+
+def create_pg_session() -> sessionmaker:
     """ create postgres db session """
     dbschema = ''
     db_user = os.environ.get("POSTGIS_DBUSER")
@@ -39,10 +33,11 @@ def create_pg_session() -> sqlalchemy.orm.sessionmaker:
         connect_args={'options': '-csearch_path={}'.format(dbschema)},
         isolation_level="READ UNCOMMITTED")
     session_maker = sessionmaker(bind=engine)
-    session = session_maker()
+    _session = session_maker()
 
     # Base.metadata.create_all(engine)
-    return session
+    return _session
+
 
 class Dataset(Base):
     """ dataset class """
@@ -75,29 +70,47 @@ class Dataset(Base):
     # first_observation = relationship("Observation", foreign_keys=[
     #                                  fk_first_observation_id])
 
-    observations = relationship(
+    # observations = relationship(
-        'Observation', back_populates='dataset', lazy=True)
+    #     'Observation', back_populates='dataset', lazy=True)
-    
+
+def new_id_factory():
+    ''' test '''
+    dbschema = ''
+    db_user = os.environ.get("POSTGIS_DBUSER")
+    db_password = os.environ.get("POSTGIS_DBPASSWORD")
+    db_url = os.environ.get("POSTGIS_DBURL")
+    engine = create_engine(
+        "postgresql+psycopg2://" + db_user + ":" + db_password + "@" + db_url,
+        connect_args={'options': '-csearch_path={}'.format(dbschema)},
+        isolation_level="READ UNCOMMITTED")
+    result = engine.execute('SELECT MAX(observation_id) FROM gba.observation')
+    mytable_max_id = result.first().max
+    if mytable_max_id is None:
+        mytable_max_id = 0
+    mytable_max_id += 1
+    return mytable_max_id
+
+
 class Observation(Base):
     """ observation class """
     __tablename__ = 'observation'
     __table_args__ = {"schema": "gba"}
 
-    id = Column('observation_id', Integer,
+    id = Column('observation_id', Integer, primary_key=True)
-                primary_key=True)
     name = Column('name', String)
-    value_type = Column('value_type', String)
+    value_type = Column('value_type', String, default="quantity")
     # pitch = Column('PITCH', String)
     # roll = Column('ROLL', String)
     sampling_time_start = Column('sampling_time_start', DateTime)
     sampling_time_end = Column('sampling_time_end', DateTime)
     result_time = Column('result_time', DateTime)
     sta_identifier = Column('sta_identifier', String)
+    value_identifier = Column('value_identifier', String)
     value_quantity = Column('value_quantity', Numeric(20, 10), nullable=False)
 
-    fk_dataset_id = Column(Integer, ForeignKey(
+    # fk_dataset_id = Column(Integer, ForeignKey(
-        'gba.dataset.dataset_id'), nullable=False)
+    #     'gba.dataset.dataset_id'), nullable=False)
-    dataset = relationship("Dataset", back_populates="observations")
+    # dataset = relationship("Dataset", back_populates="observations")
 
 
 class ObservationSchema(SQLAlchemySchema):
@@ -106,7 +119,7 @@ class ObservationSchema(SQLAlchemySchema):
     # value_quantity = fields.Integer(attribute='Value')
     # id = fields.Integer(attribute='id')
     Value = fields.Integer(attribute='value_quantity')
-    id = fields.Integer(attribute='sta_identifier')
+    id = fields.Integer(attribute='value_identifier')
     # sta_identifier= fields.String(default=uuid.uuid4()),
 
     class Meta:
@@ -134,7 +147,7 @@ class Person(Base):
             self.login, self.lname)
 
 
-class PersonSchema(SQLAlchemyAutoSchema):    
+class PersonSchema(SQLAlchemyAutoSchema):
     """ Platform class """
     class Meta:
         """ Platform class """
@@ -144,12 +157,12 @@ class PersonSchema(SQLAlchemyAutoSchema):
         #pg_session: session = create_pg_session()
         sqla_session: session = create_pg_session()
 
+
 def create_db():
-    # db_url  = 'sqlite:///db.sqlite'    
+    # db_url  = 'sqlite:///db.sqlite'
-    # engine = create_engine(db_url, echo = True )  
+    # engine = create_engine(db_url, echo = True )
     # Base.metadata.drop_all(bind=engine)
     # Base.metadata.create_all(engine)
-    
     """ create postgres db session """
     dbschema = ''
     db_user = os.environ.get("POSTGIS_DBUSER")
@@ -158,11 +171,12 @@ def create_db():
     engine = create_engine(
         "postgresql+psycopg2://" + db_user + ":" + db_password + "@" + db_url,
         connect_args={'options': '-csearch_path={}'.format(dbschema)},
-        isolation_level="READ UNCOMMITTED", echo = True)
+        isolation_level="READ UNCOMMITTED", echo=True)
     # session_maker = sessionmaker(bind=engine)
     # session = session_maker()
     Base.metadata.drop_all(bind=engine)
     Base.metadata.create_all(engine)
-        
+
+
 if __name__ == "__main__":
-    create_db()
+    create_db()

gschliefgraben_glasfaser/my_api.py

@@ -65,10 +65,10 @@ class MyApi():
     #         self.access_token = res.json()['access_token']
         # else:
         #     # Token expired -> re-authenticate
-    def getSensorData(self, sensor: string):
+    def getSensorData(self, sensor: string, date):
         ''' request observations'''
         try:
-            request = self.session.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=(\''+sensor+ '\')&start=2022-02-28&end=2022-02-28',
+            request = self.session.get('https://api.dgnss-sensors.com/gschliefgraben?sensors=(\''+sensor+ '\')&start='+date+'&end='+date,
                                        headers={
                                            'cache-control': 'no-cache',
                                            'Content-Type': 'application/x-www-form-urlencoded',

insert_sensor/execute.py

@@ -38,14 +38,25 @@ def main():
     sos_url = 'https://geomon.geologie.ac.at/52n-sos-webapp/service'
 
     ######################## Sibratsgfall
+    # offering = Offering(
+    #     "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
+    #     "sibratsgfall_3",
+    #     "Inklinometer 3, Sibratsgfaell Sensor"
+    # )
+    # procedure = Procedure( "sibratsgfall_3","sibratsgfall-3")
+    # foi = FoI("degree", "m", (47.4279288, 10.0360888, 0.0),
+    #           "sibratsgfall", "Sibratsgfall Beobachtung der Bodenbewegungen Test")
+
+     ####################### Gschliefgraben Glasfaser
     offering = Offering(
         "https://geomon.geologie.ac.at/52n-sos-webapp/api/offerings/",
-        "sibratsgfall_3",
+        "inclino1_14",
-        "Inklinometer 3, Sibratsgfaell Sensor"
+        "Inklinometer inclino1_14, Gschliefgraben Glasfaser"
     )
-    procedure = Procedure( "sibratsgfall_3","sibratsgfall-3")
+    procedure = Procedure( "inclino1_14","inclino1_14")
-    foi = FoI("degree", "m", (47.4279288, 10.0360888, 0.0),
+
-              "sibratsgfall", "Sibratsgfall Beobachtung der Bodenbewegungen Test")
+    foi = FoI("degree", "m", (47.910849, 13.774966, 0.0),
+              "FBGuard23", "Glasfaser Untersuchungen am Gschliefgraben (Gmunden)")
 
     ######################## Gschliefgraben
     # offering = Offering(
@@ -77,6 +88,13 @@ def main():
     request = requests.post(sos_url, headers=headers, json=post_data)
     print(request.text)
 
+    # {
+    # "request" : "InsertSensor",
+    # "version" : "2.0.0",
+    # "service" : "SOS",
+    # "assignedProcedure" : "inclino1_14",
+    # "assignedOffering" : "inclino1_14"
+    # }
 
 
 if __name__ == '__main__':