R
Let us assume that we have already have child data in R
stored as a data.frame or tibble. Here we copy
the longitudinal demo data maria.json from the
bdsreader package into the working directory.
success <- file.copy(system.file("examples/maria.json", package = "bdsreader"),
"maria.json", overwrite = TRUE)
The contents of the file is json format, ready for
upload:
{
"OrganisatieCode": 1234,
"Referentie": "fa308134-069e-49ce-9847-ccdae380ed6f",
"ClientGegevens": {
"Elementen": [
{
"Bdsnummer": 19,
"Waarde": "2"
},
{
"Bdsnummer": 20,
"Waarde": "20181011"
},
{
"Bdsnummer": 82,
"Waarde": "189"
},
{
"Bdsnummer": 91,
"Waarde": "2"
},
{
"Bdsnummer": 110,
"Waarde": "990"
},
{
"Bdsnummer": 238,
"Waarde": "1670"
},
{
"Bdsnummer": 240,
"Waarde": "1900"
}
],
"Groepen": [
{
"Elementen": [
{
"Bdsnummer": 63,
"Waarde": "19950704"
},
{
"Bdsnummer": 71
},
{
"Bdsnummer": 62,
"Waarde": "01"
}
]
},
{
"Elementen": [
{
"Bdsnummer": 63,
"Waarde": "19901202"
},
{
"Bdsnummer": 71
},
{
"Bdsnummer": 62,
"Waarde": "02"
}
]
}
]
},
"Contactmomenten": [
{
"Tijdstip": "20181011",
"Elementen": [
{
"Bdsnummer": 245,
"Waarde": "990"
}
]
},
{
"Tijdstip": "20181111",
"Elementen": [
{
"Bdsnummer": 235,
"Waarde": "380"
},
{
"Bdsnummer": 245,
"Waarde": "1250"
},
{
"Bdsnummer": 252,
"Waarde": "270"
}
]
},
{
"Tijdstip": "20181211",
"Elementen": [
{
"Bdsnummer": 235,
"Waarde": "435"
},
{
"Bdsnummer": 245,
"Waarde": "2100"
},
{
"Bdsnummer": 252,
"Waarde": "305"
}
]
}
]
}
There are four ways to upload the data to JAMES:
- Upload the file
"maria.json";
- Convert to a string and upload;
- Convert to a JSON object and upload;
- Read the JSON file from a URL.
The /data/upload API end point handles these cases as
follows:
# upload as file
fn <- "maria.json"
r1 <- james_post(host = host, path = "data/upload/json", txt = fn)
status_code(r1)
## [1] 201
# upload as string
js <- read_json_js(fn)
r2 <- james_post(host = host, path = "data/upload/json", txt = js)
status_code(r2)
## [1] 201
# upload as JSON object
jo <- read_json_jo(fn)
r3 <- james_post(host = host, path = "data/upload/json", txt = jo)
status_code(r3)
## [1] 201
# upload as URL
url <- file.path(host, "ocpu/library/bdsreader/examples/maria.json")
r4 <- james_post(host = host, path = "data/upload/json", txt = url)
status_code(r4)
## [1] 201
If the status is 201, the data are uploaded to JAMES and processed.
For example, the processed data after file upload is available as an R
data frame under element r1$parsed.
r1$parsed
## $psn
## id name dob dobm dobf src sex gad ga smo bw hgtm
## 1 -1 Maria 2018-10-11 1990-12-02 1995-07-04 1234 female 189 27 1 990 167
## hgtf
## 1 190
##
## $xyz
## age xname yname zname zref x y z
## 1 0.0849 age hgt hgt_z nl_2012_hgt_female_27 0.0849 38.00 -0.158
## 2 0.1670 age hgt hgt_z nl_2012_hgt_female_27 0.1670 43.50 0.047
## 3 0.0000 age wgt wgt_z nl_2012_wgt_female_27 0.0000 0.99 0.190
## 4 0.0849 age wgt wgt_z nl_2012_wgt_female_27 0.0849 1.25 -0.203
## 5 0.1670 age wgt wgt_z nl_2012_wgt_female_27 0.1670 2.10 0.015
## 6 0.0849 age hdc hdc_z nl_2012_hdc_female_27 0.0849 27.00 -0.709
## 7 0.1670 age hdc hdc_z nl_2012_hdc_female_27 0.1670 30.50 -0.913
## 8 0.0000 age bmi bmi_z nl_1997_bmi_female_nl 0.0000 NA NA
## 9 0.0849 age bmi bmi_z nl_1997_bmi_female_nl 0.0849 8.66 -5.719
## 10 0.1670 age bmi bmi_z nl_1997_bmi_female_nl 0.1670 11.10 -3.767
## 11 0.0000 hgt wfh wfh_z nl_2012_wfh_female_ NA 0.99 NA
## 12 0.0849 hgt wfh wfh_z nl_2012_wfh_female_ 38.0000 1.25 -0.001
## 13 0.1670 hgt wfh wfh_z nl_2012_wfh_female_ 43.5000 2.10 0.326
The session details, including the uploaded data, will remain
available for a limited time. After 30 minutes the session is wiped. The
session key is your entrance to the resource within the 30-minute
window. The key can be retrieved as r1$session. For
example, to see the result of the file upload session in markdown
use
(session <- r1$session)
## [1] "x07d3fd0a12b19b"
resp <- james_get(host = host, path = file.path(session, "md"))
cat(resp$parsed)
##
##
## * **psn**:
##
## -------------------------------------------------------------------------------
## id name dob dobm dobf src dnr sex gad
## ---- ------- ------------ ------------ ------------ ------ ----- -------- -----
## -1 Maria 2018-10-11 1990-12-02 1995-07-04 1234 NA female 189
## -------------------------------------------------------------------------------
##
## Table: Table continues below
##
##
## -----------------------------------------------------------------------------------
## ga smo bw hgtm hgtf agem etn pc4 blbf blbm eduf edum par
## ---- ----- ----- ------ ------ ------ ----- ----- ------ ------ ------ ------ -----
## 27 1 990 167 190 NA NA NA NA NA NA NA NA
## -----------------------------------------------------------------------------------
##
## * **xyz**:
##
## ----------------------------------------------------------------------------------
## age xname yname zname zref x y z
## -------- ------- ------- ------- ----------------------- -------- ------- --------
## 0.0849 age hgt hgt_z nl_2012_hgt_female_27 0.0849 38 -0.158
##
## 0.167 age hgt hgt_z nl_2012_hgt_female_27 0.167 43.5 0.047
##
## 0 age wgt wgt_z nl_2012_wgt_female_27 0 0.99 0.19
##
## 0.0849 age wgt wgt_z nl_2012_wgt_female_27 0.0849 1.25 -0.203
##
## 0.167 age wgt wgt_z nl_2012_wgt_female_27 0.167 2.1 0.015
##
## 0.0849 age hdc hdc_z nl_2012_hdc_female_27 0.0849 27 -0.709
##
## 0.167 age hdc hdc_z nl_2012_hdc_female_27 0.167 30.5 -0.913
##
## 0 age bmi bmi_z nl_1997_bmi_female_nl 0 NA NA
##
## 0.0849 age bmi bmi_z nl_1997_bmi_female_nl 0.0849 8.657 -5.719
##
## 0.167 age bmi bmi_z nl_1997_bmi_female_nl 0.167 11.1 -3.767
##
## 0 hgt wfh wfh_z nl_2012_wfh_female_ NA 0.99 NA
##
## 0.0849 hgt wfh wfh_z nl_2012_wfh_female_ 38 1.25 -0.001
##
## 0.167 hgt wfh wfh_z nl_2012_wfh_female_ 43.5 2.1 0.326
## ----------------------------------------------------------------------------------
##
##
## <!-- end of list -->
Troubleshooting data upload: JAMES executes checks
on the conversion and ranges of the data. To gain efficiency, it does
not automatically validate the input data against the specified JSON
schema. JAMES writes diagnostic, sometimes cryptic, messages to the
directory {session}/messages if it finds a problem. The
user can rerun the data upload with two additional flags that request
extra diagnostic output.
Example: Suppose we compromise the data by removing the required
"clientDetails" and the optional
"nestedDetails" sections. The mangled input data look
like:
{"Format":"3.0","organisationCode":12345,"reference":"Maria's mangled data","clientMeasurements":[{"bdsNumber":235,"values":[{"date":"20181111","value":380},{"date":"20181211","value":435}]}]}
Everything appears normal if we read this data by the default:
fn <- "maria-mangled.json"
r5 <- james_post(host = host, path = "data/upload/json", txt = fn)
r5$parsed
## $psn
## id name src
## 1 -1 Maria's mangled data 12345
##
## $xyz
## xname yname zname zref y
## 1 age hgt hgt_z nl_1997_hgt__nl 38
If we upload with the additional validate = TRUE flag,
JAMES runs the validation of the uploaded JSON against the JSON
schema:
r6 <- james_post(host = host, path = "data/upload/json", txt = fn, validate = TRUE)
mess <- james_get(host = host, path = file.path(r6$session, "messages"))
cat(mess$parsed)
## must have required property 'clientDetails'
which now indicates that the required JSON element
"clientDetails" is missing.
We may drill down further by setting the
intermediate = TRUE flag. This writes five JSON files that
document the data flow into {session}/files/{*}.json.
For example, we can ask for the input data as read by JAMES by
r7 <- james_post(host = host, path = "data/upload/json", txt = fn, validate = TRUE, intermediate = TRUE)
url <- file.path(host, r7$session, "files/input.json")
url
## [1] "https://james.groeidiagrammen.nl/x05e444ef5f070e/files/input.json"
With browseURL(url) we may view the file contents in the
browser. The files directory contains five JSON files:
files/input.json: the JSON input data;files/bds.json: a data frame with info per BDS
number;files/ddi.json: result of recoding BDS into GSED item
names;files/psn.json: known fixed child covariates;files/xy.json: time-varying variables.
Inspection of these files may uncover any problems with JAMES’s
understanding of the data. If needed, study the underlying R source code
at https://raw.githubusercontent.com/growthcharts/bdsreader/master/R/read_bds.R.
The validate and intermediate flag are
useful for development and debugging. In production, we recommend
leaving them at their default value (FALSE) and monitor any
messages written to {session}/messages.