How can I ingest data via a manifest and the workflow service

This article explains on how to compose a Manifest file and use it during data ingestion. In this example you will be uploading a manifest file with Well Log and a dataset.

 

Step 1: Register the Manifest DAG. Ignore if it is already registered

URL: {{osduonaws_base_url}}/api/workflow/v1/workflow
Method: POST

Body:

{
 "workflowName": "Osdu_ingest",
 "description": "This prints a storage record sent to the system",
 "registrationInstructions": {
     "concurrentWorkflowRun": 5,
     "concurrentTaskRun": 5,
     "workflowDetailContent": "",
      "active": true
 }
}

Step 2: Ingest manifest json using workflow service. The below example manifest will create Dataset for a LAS file, create well log component then associate it to an existing wellbore. The "runId" is a unique value, you can generate a new GUID or use your own randomly generated string.

URL: {{osduonaws_base_url}}/api/workflow/v1/workflow/Osdu_ingest/workflowRun
Method: POST

Body:

{
 "runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
 "executionContext": {
   "acl": {
     "owners": [
       "data.default.owners@osdu.example.com"
     ],
     "viewers": [
       "data.default.viewers@osdu.example.com"
     ]
   },
   "legal": {
     "legaltags": [
       "osdu-public-usa-dataset"
     ],
     "otherRelevantDataCountries": [
       "US"
     ]
   },
   "Payload": {
     "AppKey": "test-app",
     "data-partition-id": "osdu"
   },
   "manifest": {
     "kind": "osdu:wks:Manifest:1.0.0",
     "ReferenceData": [],
     "MasterData": [],
     "Data": {
       "WorkProduct": {
         "kind": "osdu:wks:work-product--WorkProduct:1.0.0",
         "acl": {
           "owners": [
             "data.default.owners@osdu.example.com"
           ],
           "viewers": [
             "data.default.viewers@osdu.example.com"
           ]
         },
         "legal": {
           "legaltags": [
             "osdu-public-usa-dataset"
           ],
           "otherRelevantDataCountries": [
             "US"
           ]
         },
         "data": {
           "ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
           "Name": "1013_akm11_1978_comp.las",
           "Description": "Well Log",
           "Components": [
             "surrogate-key:wpc-1"
           ]
         }
       },
       "WorkProductComponents": [
         {
           "id": "surrogate-key:wpc-1",
           "kind": "osdu:wks:work-product-component--WellLog:1.0.0",
           "acl": {
             "owners": [
               "data.default.owners@osdu.example.com"
             ],
             "viewers": [
               "data.default.viewers@osdu.example.com"
             ]
           },
           "legal": {
             "legaltags": [
               "osdu-public-usa-dataset"
             ],
             "otherRelevantDataCountries": [
               "US"
             ]
           },
           "data": {
             "ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
             "Name": "1013_akm11_1978_comp.las",
             "Description": "Well Log",
             "Datasets": [
               "surrogate-key:file-1"
             ],
             "WellboreID": "osdu:master-data--Wellbore:1013:",
             "TopMeasuredDepth": 2182.0004,
             "BottomMeasuredDepth": 2481,
             "Curves": [
               {
                 "Mnemonic": "DEPT",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:M:"
               },
               {
                 "Mnemonic": "GR",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:GAPI:"
               },
               {
                 "Mnemonic": "DT",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:US%2FF:"
               },
               {
                 "Mnemonic": "RHOB",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
               },
               {
                 "Mnemonic": "DRHO",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
               },
               {
                 "Mnemonic": "NPHI",
                 "TopDepth": 2182,
                 "BaseDepth": 2481,
                 "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                 "CurveUnit": "osdu:reference-data--UnitOfMeasure:V%2FV:"
               }
             ]
           }
         }
       ],
       "Datasets": [
         {
           "id": "surrogate-key:file-1",
           "kind": "osdu:wks:dataset--File.Generic:1.0.0",
           "acl": {
             "owners": [
               "data.default.owners@osdu.example.com"
             ],
             "viewers": [
               "data.default.viewers@osdu.example.com"
             ]
           },
           "legal": {
             "legaltags": [
               "osdu-public-usa-dataset"
             ],
             "otherRelevantDataCountries": [
               "US"
             ]
           },
           "data": {
             "ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
             "SchemaFormatTypeID": "osdu:reference-data--SchemaFormatType:LAS2:",
             "DatasetProperties": {
               "FileSourceInfo": {
                 "FileSource": "s3://osdu-artifacts-911600437027-us-east-1/r1/data/provided/well-logs/1013_akm11_1978_comp.las",
                 "Name": "1013_akm11_1978_comp.las",
                 "PreloadFilePath": "s3://osdu-artifacts-911600437027-us-east-1/r1/data/provided/well-logs/1013_akm11_1978_comp.las"
               }
             }
           }
         }
       ]
     }
   }
 }
}

Once it is submitted into OSDU platform it will return the runId like below. We will be using this runId to retrieve the workflow status.

{
   "workflowId": "osdu:Osdu_ingest",
   "runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
   "startTimeStamp": 1649586370726,
   "status": "submitted",
   "submittedBy": "admin@testing.com"
}

Step 3: Check the status of the triggered workflow by passing the runId

URL: {{osduonaws_base_url}}/api/workflow/v1/workflow/Osdu_ingest/workflowRun/309a5b3e-0c95-4d62-85ee-a73cd132b3df
Metho: GET

The above GET call will return a response like below

{
   "workflowId": "osdu:Osdu_ingest",
   "runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
   "startTimeStamp": 1649586370726,
   "endTimeStamp": 1649586423828,
   "status": "finished",
   "submittedBy": "admin@testing.com"
}

 

Step 4: To verify the newly created records login to Airflow then click on Osdu_ingest DAG

             a. Click on “Graph View” and select the Run Id from the drop-down list

airflow-graph.png

Note: Though the Airflow run id status shows success we have to make sure that there are no errors by going through the logs of each step.

          b. To find the newly created records ids follow the below steps

           i. Click on “process_single_manifest_file_task”

airflow-log.png

           ii. From the pop-up click on Log button. This action will show the complete log

           iii. To see the exact record ids, select XCom as shown below

xcom.png

Step 5: Use the above output Ids in search service to get the complete record details. For example, to retrieve the work product component follow the below steps

URL: {{osduonaws_base_url}}/api/search/v2/query
Method: POST

Body:

{
   "kind": "*:*:*:*",
   "query": "id:\"osdu:work-product-component--WellLog:1187997cb1d74d028e8b02424815df23\""
   }
}

The above call will return well log details as below

{
   "results": [
       {
           "data": {
                "SpatialArea.QuantitativeAccuracyBandID": null,
               "Description": "Well Log",
                "ResourceLifecycleStatus": null,
                "SpatialArea.SpatialParameterTypeID": null,
               "TopMeasuredDepth": 2182.0004,
                "ServiceCompanyID": null,
                "ResourceCurationStatus": null,
                "SpatialArea.SpatialGeometryTypeID": null,
                "Source": null,
                "IsExtendedLoad": null,
               "Name": "1013_akm11_1978_comp.las",
               "Datasets": [
                   "osdu:dataset--File.Generic:ee5d32cdae804bc4b0d4b0425c944c61:"
               ],
                "VerticalMeasurement.VerticalCRSID": null,
                "LogVersion": null,
                "VerticalMeasurement.VerticalMeasurementTypeID": null,
                "LogSource": null,
                "ToolStringDescription": null,
                "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
                "DrillingFluidProperty": null,
               "ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
                "VerticalMeasurement.WellboreTVDTrajectoryID": null,
                "VerticalMeasurement.VerticalMeasurementPathID": null,
                "ExistenceKind": null,
                "HoleTypeLogging": null,
                "VerticalMeasurement.VerticalMeasurementDescription": null,
                "LoggingDirection": null,
                "SpatialPoint.SpatialParameterTypeID": null,
                "VerticalMeasurementID": null,
               "WellboreID": "osdu:master-data--Wellbore:1013:",
                "ActivityType": null,
                "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
                "SpatialPoint.QuantitativeAccuracyBandID": null,
                "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
                "SpatialArea.CoordinateQualityCheckPerformedBy": null,
                "SpatialPoint.SpatialGeometryTypeID": null,
                "ResourceHomeRegionID": null,
                "VerticalMeasurement.VerticalMeasurementSourceID": null,
                "WellLogTypeID": null,
                "IsDiscoverable": null,
                "LoggingService": null,
               "BottomMeasuredDepth": 2481.0,
                "VerticalMeasurement.VerticalReferenceID": null,
                "SubmitterName": null,
                "LogRun": null,
                "SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
                "LogActivity": null,
               "Curves": [
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "DEPT",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:M:"
                   },
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "GR",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:GAPI:"
                   },
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "DT",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:US%2FF:"
                   },
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "RHOB",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
                   },
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "DRHO",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
                   },
                   {
                        "IsProcessed": null,
                        "LogCurveMainFamilyID": null,
                        "LogCurveFamilyID": null,
                        "CurveID": null,
                       "TopDepth": 2182.0,
                        "CurveVersion": null,
                        "InterpreterName": null,
                        "CurveQuality": null,
                        "NullValue": null,
                        "Interpolate": null,
                       "DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
                        "DepthCoding": null,
                       "Mnemonic": "NPHI",
                       "BaseDepth": 2481.0,
                        "LogCurveTypeID": null,
                        "LogCurveBusinessValueID": null,
                       "CurveUnit": "osdu:reference-data--UnitOfMeasure:V%2FV:"
                   }
               ]
           },
           "kind": "osdu:wks:work-product-component--WellLog:1.0.0",
           "source": "wks",
           "acl": {
               "viewers": [
                   "data.default.viewers@osdu.example.com"
               ],
               "owners": [
                   "data.default.owners@osdu.example.com"
               ]
           },
           "type": "work-product-component--WellLog",
           "version": 1649586418692664,
            "tags": null,
           "createTime": "2022-04-10T10:26:59.091Z",
           "authority": "osdu",
           "namespace": "osdu:wks",
           "legal": {
               "legaltags": [
                   "osdu-public-usa-dataset"
               ],
               "otherRelevantDataCountries": [
                   "US"
               ],
               "status": "compliant"
           },
           "createUser": "serviceprincipal@testing.com",
           "id": "osdu:work-product-component--WellLog:1187997cb1d74d028e8b02424815df23"
       }
   ],
    "aggregations": null,
   "totalCount": 1
}

 

 

ste

Have more questions? Submit a request

0 Comments

Please sign in to leave a comment.
Powered by Zendesk