This article explains on how to compose a Manifest file and use it during data ingestion. In this example you will be uploading a manifest file with Well Log and a dataset.
Step 1: Register the Manifest DAG. Ignore if it is already registered
URL: {{osduonaws_base_url}}/api/workflow/v1/workflow
Method: POST
Body:
{
"workflowName": "Osdu_ingest",
"description": "This prints a storage record sent to the system",
"registrationInstructions": {
"concurrentWorkflowRun": 5,
"concurrentTaskRun": 5,
"workflowDetailContent": "",
"active": true
}
}
Step 2: Ingest manifest json using workflow service. The below example manifest will create Dataset for a LAS file, create well log component then associate it to an existing wellbore. The "runId" is a unique value, you can generate a new GUID or use your own randomly generated string.
URL: {{osduonaws_base_url}}/api/workflow/v1/workflow/Osdu_ingest/workflowRun
Method: POST
Body:
{
"runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
"executionContext": {
"acl": {
"owners": [
"data.default.owners@osdu.example.com"
],
"viewers": [
"data.default.viewers@osdu.example.com"
]
},
"legal": {
"legaltags": [
"osdu-public-usa-dataset"
],
"otherRelevantDataCountries": [
"US"
]
},
"Payload": {
"AppKey": "test-app",
"data-partition-id": "osdu"
},
"manifest": {
"kind": "osdu:wks:Manifest:1.0.0",
"ReferenceData": [],
"MasterData": [],
"Data": {
"WorkProduct": {
"kind": "osdu:wks:work-product--WorkProduct:1.0.0",
"acl": {
"owners": [
"data.default.owners@osdu.example.com"
],
"viewers": [
"data.default.viewers@osdu.example.com"
]
},
"legal": {
"legaltags": [
"osdu-public-usa-dataset"
],
"otherRelevantDataCountries": [
"US"
]
},
"data": {
"ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
"Name": "1013_akm11_1978_comp.las",
"Description": "Well Log",
"Components": [
"surrogate-key:wpc-1"
]
}
},
"WorkProductComponents": [
{
"id": "surrogate-key:wpc-1",
"kind": "osdu:wks:work-product-component--WellLog:1.0.0",
"acl": {
"owners": [
"data.default.owners@osdu.example.com"
],
"viewers": [
"data.default.viewers@osdu.example.com"
]
},
"legal": {
"legaltags": [
"osdu-public-usa-dataset"
],
"otherRelevantDataCountries": [
"US"
]
},
"data": {
"ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
"Name": "1013_akm11_1978_comp.las",
"Description": "Well Log",
"Datasets": [
"surrogate-key:file-1"
],
"WellboreID": "osdu:master-data--Wellbore:1013:",
"TopMeasuredDepth": 2182.0004,
"BottomMeasuredDepth": 2481,
"Curves": [
{
"Mnemonic": "DEPT",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:M:"
},
{
"Mnemonic": "GR",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:GAPI:"
},
{
"Mnemonic": "DT",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:US%2FF:"
},
{
"Mnemonic": "RHOB",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
},
{
"Mnemonic": "DRHO",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
},
{
"Mnemonic": "NPHI",
"TopDepth": 2182,
"BaseDepth": 2481,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"CurveUnit": "osdu:reference-data--UnitOfMeasure:V%2FV:"
}
]
}
}
],
"Datasets": [
{
"id": "surrogate-key:file-1",
"kind": "osdu:wks:dataset--File.Generic:1.0.0",
"acl": {
"owners": [
"data.default.owners@osdu.example.com"
],
"viewers": [
"data.default.viewers@osdu.example.com"
]
},
"legal": {
"legaltags": [
"osdu-public-usa-dataset"
],
"otherRelevantDataCountries": [
"US"
]
},
"data": {
"ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
"SchemaFormatTypeID": "osdu:reference-data--SchemaFormatType:LAS2:",
"DatasetProperties": {
"FileSourceInfo": {
"FileSource": "s3://osdu-artifacts-911600437027-us-east-1/r1/data/provided/well-logs/1013_akm11_1978_comp.las",
"Name": "1013_akm11_1978_comp.las",
"PreloadFilePath": "s3://osdu-artifacts-911600437027-us-east-1/r1/data/provided/well-logs/1013_akm11_1978_comp.las"
}
}
}
}
]
}
}
}
}
Once it is submitted into OSDU platform it will return the runId like below. We will be using this runId to retrieve the workflow status.
{
"workflowId": "osdu:Osdu_ingest",
"runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
"startTimeStamp": 1649586370726,
"status": "submitted",
"submittedBy": "admin@testing.com"
}
Step 3: Check the status of the triggered workflow by passing the runId
URL: {{osduonaws_base_url}}/api/workflow/v1/workflow/Osdu_ingest/workflowRun/309a5b3e-0c95-4d62-85ee-a73cd132b3df
Metho: GET
The above GET call will return a response like below
{
"workflowId": "osdu:Osdu_ingest",
"runId": "309a5b3e-0c95-4d62-85ee-a73cd132b3df",
"startTimeStamp": 1649586370726,
"endTimeStamp": 1649586423828,
"status": "finished",
"submittedBy": "admin@testing.com"
}
Step 4: To verify the newly created records login to Airflow then click on Osdu_ingest DAG
a. Click on “Graph View” and select the Run Id from the drop-down list
Note: Though the Airflow run id status shows success we have to make sure that there are no errors by going through the logs of each step.
b. To find the newly created records ids follow the below steps
i. Click on “process_single_manifest_file_task”
ii. From the pop-up click on Log button. This action will show the complete log
iii. To see the exact record ids, select XCom as shown below
Step 5: Use the above output Ids in search service to get the complete record details. For example, to retrieve the work product component follow the below steps
URL: {{osduonaws_base_url}}/api/search/v2/query
Method: POST
Body:
{
"kind": "*:*:*:*",
"query": "id:\"osdu:work-product-component--WellLog:1187997cb1d74d028e8b02424815df23\""
}
}
The above call will return well log details as below
{
"results": [
{
"data": {
"SpatialArea.QuantitativeAccuracyBandID": null,
"Description": "Well Log",
"ResourceLifecycleStatus": null,
"SpatialArea.SpatialParameterTypeID": null,
"TopMeasuredDepth": 2182.0004,
"ServiceCompanyID": null,
"ResourceCurationStatus": null,
"SpatialArea.SpatialGeometryTypeID": null,
"Source": null,
"IsExtendedLoad": null,
"Name": "1013_akm11_1978_comp.las",
"Datasets": [
"osdu:dataset--File.Generic:ee5d32cdae804bc4b0d4b0425c944c61:"
],
"VerticalMeasurement.VerticalCRSID": null,
"LogVersion": null,
"VerticalMeasurement.VerticalMeasurementTypeID": null,
"LogSource": null,
"ToolStringDescription": null,
"SpatialPoint.CoordinateQualityCheckPerformedBy": null,
"DrillingFluidProperty": null,
"ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:",
"VerticalMeasurement.WellboreTVDTrajectoryID": null,
"VerticalMeasurement.VerticalMeasurementPathID": null,
"ExistenceKind": null,
"HoleTypeLogging": null,
"VerticalMeasurement.VerticalMeasurementDescription": null,
"LoggingDirection": null,
"SpatialPoint.SpatialParameterTypeID": null,
"VerticalMeasurementID": null,
"WellboreID": "osdu:master-data--Wellbore:1013:",
"ActivityType": null,
"VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
"SpatialPoint.QuantitativeAccuracyBandID": null,
"SpatialArea.QualitativeSpatialAccuracyTypeID": null,
"SpatialArea.CoordinateQualityCheckPerformedBy": null,
"SpatialPoint.SpatialGeometryTypeID": null,
"ResourceHomeRegionID": null,
"VerticalMeasurement.VerticalMeasurementSourceID": null,
"WellLogTypeID": null,
"IsDiscoverable": null,
"LoggingService": null,
"BottomMeasuredDepth": 2481.0,
"VerticalMeasurement.VerticalReferenceID": null,
"SubmitterName": null,
"LogRun": null,
"SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
"LogActivity": null,
"Curves": [
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "DEPT",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:M:"
},
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "GR",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:GAPI:"
},
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "DT",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:US%2FF:"
},
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "RHOB",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
},
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "DRHO",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:G%2FC3:"
},
{
"IsProcessed": null,
"LogCurveMainFamilyID": null,
"LogCurveFamilyID": null,
"CurveID": null,
"TopDepth": 2182.0,
"CurveVersion": null,
"InterpreterName": null,
"CurveQuality": null,
"NullValue": null,
"Interpolate": null,
"DepthUnit": "osdu:reference-data--UnitOfMeasure:M:",
"DepthCoding": null,
"Mnemonic": "NPHI",
"BaseDepth": 2481.0,
"LogCurveTypeID": null,
"LogCurveBusinessValueID": null,
"CurveUnit": "osdu:reference-data--UnitOfMeasure:V%2FV:"
}
]
},
"kind": "osdu:wks:work-product-component--WellLog:1.0.0",
"source": "wks",
"acl": {
"viewers": [
"data.default.viewers@osdu.example.com"
],
"owners": [
"data.default.owners@osdu.example.com"
]
},
"type": "work-product-component--WellLog",
"version": 1649586418692664,
"tags": null,
"createTime": "2022-04-10T10:26:59.091Z",
"authority": "osdu",
"namespace": "osdu:wks",
"legal": {
"legaltags": [
"osdu-public-usa-dataset"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
},
"createUser": "serviceprincipal@testing.com",
"id": "osdu:work-product-component--WellLog:1187997cb1d74d028e8b02424815df23"
}
],
"aggregations": null,
"totalCount": 1
}
ste
0 Comments