`,
+ 'log:check': `Devtools checks`
+ }});
diff --git a/.github/workflows/dsBaseClient_test_suite.yaml b/.github/workflows/dsBaseClient_test_suite.yaml
index b8a6f3ccf..a33032abe 100644
--- a/.github/workflows/dsBaseClient_test_suite.yaml
+++ b/.github/workflows/dsBaseClient_test_suite.yaml
@@ -1,244 +1,839 @@
################################################################################
+# trigger CI
# DataSHIELD GHA test suite - dsBaseClient
-# Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz
+# Replaces azure-pipelines.yml / opal_azure-pipelines.yml / armadillo_azure-pipelines.yml.
#
-# Inside the root directory $(Pipeline.Workspace) will be a file tree like:
-# /dsBaseClient <- Checked out version of datashield/dsBaseClient
-# /dsBaseClient/logs <- Where results of tests and logs are collated
-# /testStatus <- Checked out version of datashield/testStatus
+# This is one of three separate workflow files, each its own named check on a
+# PR/run: this one (all the actual test execution), check.yaml (doc-sync +
+# R CMD check), lint.yaml (lintr). Split so each shows as its own category
+# rather than one graph mixing test execution with static checks.
#
-# As of Sept. 2025 this takes ~ 95 mins to run.
+# Structure (all jobs below run in parallel except where "needs" says otherwise):
+# opal-dsbase (matrix x8) - dsBase suite against Opal, one shard per
+# category, plus a dsdanger entry, all grouped under
+# one summary box.
+# opal-report - needs opal-dsbase; merges results, computes its own
+# pass/fail, posts its own row into the shared PR
+# comment (see .github/scripts/post-ci-comment.js).
+# armadillo-dsbase (matrix x8) - dsBase suite against Armadillo, same split.
+# armadillo-report - needs armadillo-dsbase; same as opal-report, plus
+# coverage (computed here only - see comment at that
+# step for why one backend's figure is sufficient).
+#
+# There is no separate combining/summary job: dsBaseClient's own R/ source
+# never branches on backend, so Armadillo and Opal results are independently
+# meaningful and each report job stands alone (gate on both being required
+# checks in branch protection, rather than one job that waits on both).
+#
+# The dsBase suite (matching TEST_FILTER_DSBASE - same 292 files the Azure
+# pipelines run) is split into 7 shards - smk (116 files, 2 shards), perf (51
+# fixed 30s-loop benchmarks, 3 shards - by far the slowest per-file), arg (1
+# shard), and misc (1 shard) - each shard with its own testthat filter
+# substring. smk/perf are split by the first letter of the function name
+# (after any "ds." prefix) rather than an enumerated file list, so newly
+# added test files fall into a bucket automatically. The small dsDanger suite
+# is folded in as an 8th matrix entry (steps gated on matrix.category ==
+# 'dsdanger') rather than a standalone job, so it groups under the same
+# summary box instead of its own. This is one job with a matrix (not split
+# into separate job definitions per category) so all entries share one
+# summary box in the run graph, and so they don't share a reusable workflow
+# name - GitHub's default same-name concurrency cap of 2 would otherwise
+# throttle them to 2-at-a-time. Setup (checkout through installing dsBase)
+# lives in a shared composite action (setup-armadillo-with-dsbase /
+# setup-opal-with-dsbase), used by every matrix entry including dsdanger, so
+# that part stays DRY.
+#
+# Each dsbase/dsdanger job spins up its OWN backend instance (isolated - no
+# shared server state / concurrency risk between categories running at once).
+#
+# Opal runs via docker-compose (docker-compose_opal.yml).
+# Armadillo runs as a plain `java -jar` process (not docker-compose): Armadillo
+# self-manages its Rock container over the host Docker socket
+# (docker-management-enabled: true / docker-run-in-container: false), which skips
+# building/pulling the old custom armadillo_citest image. See
+# molgenis-service-armadillo's application.template.yml for that flag pairing.
+#
+# Every job starts its backend as the very first step so it boots in the
+# background while R dependencies install, instead of paying for both serially.
+#
+# As of Sept. 2025 the single-backend, dsBase-only, unsharded version of this
+# took ~ 95 mins; the full (Opal+Armadillo, dsBase+dsDanger) unsharded run is
+# well over an hour.
################################################################################
name: dsBaseClient tests' suite
on:
push:
+ branches: [main, master, 'v*-dev']
+ pull_request:
+ workflow_dispatch:
+ inputs:
+ dsbase-ref:
+ description: dsBase branch, tag or SHA to test against
+ required: false
+ default: v7.0-dev
schedule:
- cron: '0 0 * * 0' # Weekly
- cron: '0 1 * * *' # Nightly
+# A new push to the same ref supersedes any run still in progress for it, so
+# we don't burn compute on stale commits. Scoped by event_name too, so a
+# schedule/workflow_dispatch run is never auto-cancelled by an unrelated push.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
+
+permissions:
+ contents: read
+
+env:
+ _r_check_system_clock_: 0
+ PROJECT_NAME: dsBaseClient
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+ DSBASE_REF: ${{ inputs.dsbase-ref || 'v7.0-dev' }}
+ R_KEEP_PKG_SOURCE: yes
+ # Selects perf_files/__perf-profile.csv as the perf
+ # test reference rates/tolerances (see tests/testthat/perf_tests/perf_rate.R).
+ # Reuses the existing azure-pipeline reference files rather than adding new
+ # ones, matching what the old Azure pipelines set via perf.profile.
+ PERF_PROFILE: azure-pipeline
+ # dsBase shards get their filter from matrix.filter per entry instead.
+ TEST_FILTER_DSDANGER: '__dgr-|datachk_dgr-|smk_dgr-|arg_dgr-|disc_dgr-|smk_expt_dgr-|expt_dgr-|math_dgr-'
+
jobs:
- dsBaseClient_test_suite:
+
+ # Runs immediately (no `needs`) so the two test rows reset to pending as
+ # soon as a new commit lands, rather than showing the previous commit's
+ # result for the ~20-60 min the matrix jobs take to complete.
+ mark-pending:
+ name: Mark tests pending
runs-on: ubuntu-latest
- timeout-minutes: 120
+ timeout-minutes: 5
permissions:
contents: read
+ pull-requests: write
+ steps:
+ - name: Checkout dsBaseClient
+ uses: actions/checkout@v5
+ with:
+ path: dsBaseClient
+
+ - name: Mark pending
+ uses: actions/github-script@v8
+ with:
+ script: |
+ const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js');
+ await postCiComment({ github, context, updates: {
+ 'row:tests-armadillo': `
Armadillo unit tests
⏳ pending
`,
+ 'row:tests-opal': `
Opal unit tests
⏳ pending
`,
+ 'row:coverage': `
Test coverage
⏳ pending
`,
+ 'ver:armadillo': `_pending_`,
+ 'ver:opal': `_pending_`,
+ 'log:tests-armadillo': `_pending_`,
+ 'log:tests-opal': `_pending_`,
+ 'log:coverage': `_pending_`
+ }});
- # These should all be constant, except TEST_FILTER. This can be used to test
- # subsets of test files in the testthat directory. Options are like:
- # '*' <- Run all tests.
- # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests.
- # '*_smk_*' <- Run all the smoke tests for all functions.
+ ################################################################################
+ # Opal - dsBase suite, sharded, plus dsDanger folded in as an extra matrix
+ # entry. Each entry is a fully isolated job with its own Opal instance.
+ ################################################################################
+ # One job, 8-entry matrix, so all entries group under a single summary box
+ # in the run graph (previously split into 3 category-group jobs calling a
+ # reusable workflow - reverted since that lost the combined summary and,
+ # worse, made all calls share the reusable workflow's name, which triggers
+ # GitHub's default same-name concurrency cap of 2 and throttled the shards
+ # to 2-at-a-time instead of running in parallel).
+ opal-dsbase:
+ name: Opal tests (${{ matrix.category }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ # smk and perf are split by the first letter of the function name
+ # (after any "ds." prefix) rather than an enumerated file list, so
+ # newly added test files fall into a bucket automatically. dsdanger is
+ # a separate, smaller suite (steps below are gated on
+ # matrix.category == 'dsdanger') folded in here so it groups under this
+ # job's summary box instead of a standalone job.
+ matrix:
+ include:
+ - category: smk-1
+ filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)'
+ - category: smk-2
+ filter: 'smk-ds.[m-vM-V]'
+ - category: arg
+ filter: 'arg-'
+ - category: perf-1
+ filter: 'perf-ds.[a-cA-C]'
+ - category: perf-2
+ filter: 'perf-ds.[d-mD-M]'
+ - category: perf-3
+ filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)'
+ - category: misc
+ filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-'
+ - category: dsdanger
env:
- TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-'
- _r_check_system_clock_: 0
- WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }}
PROJECT_NAME: dsBaseClient
- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
- REPO_OWNER: ${{ github.repository_owner }}
- R_KEEP_PKG_SOURCE: yes
- GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }}
-
+ DS_DRIVER: OpalDriver
+ DSDANGER_REF: '6.3.4'
steps:
- name: Checkout dsBaseClient
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
path: dsBaseClient
- - name: Checkout testStatus
- if: ${{ github.actor != 'nektos/act' }} # for local deployment only
- uses: actions/checkout@v4
+ - uses: ./dsBaseClient/.github/actions/setup-opal-with-dsbase
with:
- repository: ${{ env.REPO_OWNER }}/testStatus
- ref: master
- path: testStatus
- persist-credentials: false
- token: ${{ env.GITHUB_TOKEN }}
+ dsbase-ref: ${{ env.DSBASE_REF }}
+
+ - name: Install dsDangerClient
+ if: matrix.category == 'dsdanger'
+ run: |
+ R -q -e "
+ ref <- Sys.getenv('BRANCH_NAME')
+ ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE)
+ if (!ok) pak::pkg_install('github::datashield/dsDangerClient')"
+
+ - name: Install dsDanger package on Opal server
+ if: matrix.category == 'dsdanger'
+ run: |
+ R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)"
+ R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsDanger', username = 'datashield', ref = '${{ env.DSDANGER_REF }}'); opal.logout(opal)"
+ working-directory: dsBaseClient
+
+ - name: Run dsBase tests with JUnit report
+ if: matrix.category != 'dsdanger'
+ run: |
+ R -q -e '
+ devtools::load_all(quiet = TRUE);
+ library(testthat);
+ output_file <- file("test_console_output_dsbase.txt");
+ sink(output_file, split = TRUE);
+ junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml"));
+ progress_rep <- ProgressReporter$new(max_failures = 999999);
+ multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
+ options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
+ test_dir("tests/testthat", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$?
+ cat test_console_output_dsbase.txt
+ n_tests=$(grep -c ' entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass."
+ R_EXIT=1
+ fi
+ n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsbase.xml | wc -l | tr -d ' ')
+ if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then
+ echo "$n_failed test failure(s)/error(s) in test_results_dsbase.xml - failing this shard."
+ R_EXIT=1
+ fi
+ exit "${R_EXIT:-0}"
+ working-directory: dsBaseClient
+
+ - name: Run dsDanger tests with JUnit report
+ if: matrix.category == 'dsdanger'
+ run: |
+ R -q -e '
+ devtools::load_all(quiet = TRUE);
+ library(testthat);
+ output_file <- file("test_console_output_dsdanger.txt");
+ sink(output_file, split = TRUE);
+ junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml"));
+ progress_rep <- ProgressReporter$new(max_failures = 999999);
+ multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
+ options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
+ test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$?
+ cat test_console_output_dsdanger.txt
+ n_tests=$(grep -c ' entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass."
+ R_EXIT=1
+ fi
+ n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsdanger.xml | wc -l | tr -d ' ')
+ if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then
+ echo "$n_failed test failure(s)/error(s) in test_results_dsdanger.xml - failing this shard."
+ R_EXIT=1
+ fi
+ exit "${R_EXIT:-0}"
+ working-directory: dsBaseClient
- - name: Uninstall default MySQL
+ # Written regardless of outcome so opal-report can tell a shard that
+ # never reported (crashed in setup, before any test XML existed) apart
+ # from one that reported 0 failures - job.status already reflects the
+ # test step's exit code by this point. mkdir -p so this still succeeds
+ # even if checkout itself is what failed. Written under dsBaseClient/,
+ # alongside everything else in the same upload below, so upload-artifact
+ # doesn't widen its common-ancestor computation to the workspace root
+ # and shift every other file down an extra directory level.
+ - name: Record shard outcome
+ if: always()
run: |
- curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
- sudo service mysql stop || true
- sudo apt-get update
- sudo apt-get remove --purge mysql-client mysql-server mysql-common -y
- sudo apt-get autoremove -y
- sudo apt-get autoclean -y
- sudo rm -rf /var/lib/mysql/
+ mkdir -p dsBaseClient
+ echo "${{ job.status }}" > dsBaseClient/shard_status.txt
+
+ - name: Upload shard results
+ if: always() && matrix.category != 'dsdanger'
+ uses: actions/upload-artifact@v6
+ with:
+ name: opal-dsbase-${{ matrix.category }}
+ path: |
+ dsBaseClient/test_results_dsbase.xml
+ dsBaseClient/test_console_output_dsbase.txt
+ dsBaseClient/tests/testthat/data_files/dsbase_version.txt
+ dsBaseClient/shard_status.txt
+
+ - name: Upload dsDanger results
+ if: always() && matrix.category == 'dsdanger'
+ uses: actions/upload-artifact@v6
+ with:
+ name: opal-dsdanger
+ path: |
+ dsBaseClient/test_results_dsdanger.xml
+ dsBaseClient/test_console_output_dsdanger.txt
+ dsBaseClient/shard_status.txt
- - uses: r-lib/actions/setup-pandoc@v2
+
+ ################################################################################
+ # Opal - merge all matrix entry results and publish the report.
+ ################################################################################
+ opal-report:
+ name: Opal report
+ needs: [opal-dsbase]
+ if: always()
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ pull-requests: write
+ steps:
+ - name: Checkout dsBaseClient
+ uses: actions/checkout@v5
+ with:
+ path: dsBaseClient
- uses: r-lib/actions/setup-r@v2
with:
r-version: release
- http-user-agent: release
use-public-rspm: true
- - name: Install R and dependencies
- run: |
- sudo apt-get install --no-install-recommends software-properties-common dirmngr -y
- wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
- sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
- sudo apt-get update -qq
- sudo apt-get upgrade -y
- sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y
- sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y
- sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')"
- sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))"
-
- uses: r-lib/actions/setup-r-dependencies@v2
+ env:
+ PKG_INCLUDE_LINKINGTO: true
with:
- dependencies: 'c("Imports")'
+ working-directory: dsBaseClient
+ dependencies: 'c("Depends", "Imports", "LinkingTo")'
extra-packages: |
- any::rcmdcheck
- cran::devtools
- cran::git2r
- cran::RCurl
- cran::readr
- cran::magrittr
cran::xml2
- cran::purrr
- cran::dplyr
- cran::stringr
- cran::tidyr
- cran::quarto
- cran::knitr
- cran::kableExtra
- cran::rmarkdown
- cran::downlit
- needs: check
-
- - name: Check manual updated
+
+ - name: Download shard/dsdanger results
+ uses: actions/download-artifact@v7
+ with:
+ pattern: 'opal-*'
+ path: dsBaseClient/artifacts
+
+ - name: Merge JUnit results
run: |
- orig_sum=$(find man -type f | sort -u | xargs cat | md5sum)
- R -q -e "devtools::document()"
- new_sum=$(find man -type f | sort -u | xargs cat | md5sum)
- if [ "$orig_sum" != "$new_sum" ]; then
- echo "Your committed man/*.Rd files are out of sync with the R headers."
- exit 1
- fi
+ mkdir -p logs
+ cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt
+
+ Rscript -e '
+ xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE)
+ docs <- lapply(xml_files, xml2::read_xml)
+ root <- xml2::xml_new_root("testsuites")
+ for (doc in docs) {
+ for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s)
+ }
+ xml2::write_xml(root, "logs/test_results.xml")
+ '
working-directory: dsBaseClient
- continue-on-error: true
- - name: Devtools checks
+ - name: Upload merged results
+ uses: actions/upload-artifact@v6
+ with:
+ name: opal-report-results
+ path: |
+ dsBaseClient/logs/test_results.xml
+ dsBaseClient/logs/test_console_output.txt
+
+ - name: Compute results & write summary
+ id: results
+ env:
+ OPAL_DSBASE_RESULT: ${{ needs.opal-dsbase.result }}
run: |
- R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout
- grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout
- working-directory: dsBaseClient
- continue-on-error: true
+ Rscript -e '
+ source(".github/scripts/summarise-junit.R")
+ res <- summarise_junit("logs/test_results.xml", "Opal", "artifacts")
+ writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY"))
+ cat(res$summary, sep = "\n")
+
+ version <- find_dsbase_version("artifacts")
+
+ # needs.opal-dsbase.result is "failure" if ANY matrix shard did not
+ # succeed (even if the shards that DID upload results show 0
+ # failures) - a shard that never reported must not look like a pass.
+ shard_ok <- Sys.getenv("OPAL_DSBASE_RESULT") == "success"
+ ok <- res$ok && shard_ok
+ # Only a fallback: the usual case (a shard crashed/reported 0
+ # tests) is already named above via res$shard_problems - this
+ # covers the rare gap where a shard job failed without leaving
+ # any trace summarise_junit() could identify.
+ if (!shard_ok && length(res$shard_problems) == 0) {
+ message("One or more Opal dsbase/dsdanger matrix entries did not succeed, but none could be identified from their uploaded results.")
+ }
+
+ out <- Sys.getenv("GITHUB_OUTPUT")
+ cat(
+ sprintf("ok=%s\n", tolower(ok)),
+ sprintf("tally=%s\n", res$tally),
+ sprintf("version=%s\n", version),
+ file = out, append = TRUE, sep = ""
+ )
- - name: Start Armadillo docker-compose
- run: docker compose -f docker-compose_armadillo.yml up -d --build
+ if (!ok) message("Opal tests failed.")
+ quit(save = "no", status = if (ok) 0 else 1)
+ '
working-directory: dsBaseClient
- - name: Install test datasets
+ - name: Post PR comment
+ if: always()
+ uses: actions/github-script@v8
+ with:
+ script: |
+ // steps.results.outputs.* come back empty (not "true"/"false") if
+ // that step never got far enough to write them - e.g. it errored
+ // or was skipped outright because an earlier step failed. Fall
+ // back to something informative rather than a blank tally.
+ const ok = '${{ steps.results.outputs.ok }}' === 'true';
+ const tally = '${{ steps.results.outputs.tally }}' || 'error - see log';
+ const version = '${{ steps.results.outputs.version }}' || 'unknown';
+ // #summary- scrolls straight to this job's summary card
+ // (the pass/fail tally written via GITHUB_STEP_SUMMARY) instead
+ // of just the top of the run page.
+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}#summary-${{ job.check_run_id }}`;
+
+ const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js');
+ await postCiComment({ github, context, updates: {
+ 'row:tests-opal': `
Opal unit tests
${ok ? '✅' : '❌'} ${tally}
`,
+ 'ver:opal': `\`${version}\``,
+ 'log:tests-opal': `Opal unit tests`
+ }});
+
+
+ ################################################################################
+ # Armadillo - dsBase suite, sharded 4 ways. Each shard downloads and runs its
+ # own Armadillo jar instance.
+ #
+ # Runs as a plain `java -jar` process instead of docker-compose: Armadillo
+ # self-manages its own Rock container over the host Docker socket
+ # (docker-management-enabled: true / docker-run-in-container: false), which
+ # avoids building/pulling the old custom armadillo_citest image and its
+ # dockerised Armadillo layer. The latest GitHub release jar is downloaded at
+ # run time. No process is restarted after installing dsBase - install then
+ # whitelist directly.
+ ################################################################################
+ # One job, 8-entry matrix, so all entries group under a single summary box
+ # in the run graph (previously split into 3 category-group jobs calling a
+ # reusable workflow - reverted since that lost the combined summary and,
+ # worse, made all calls share the reusable workflow's name, which triggers
+ # GitHub's default same-name concurrency cap of 2 and throttled the shards
+ # to 2-at-a-time instead of running in parallel).
+ armadillo-dsbase:
+ name: Armadillo tests (${{ matrix.category }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ # smk and perf are split by the first letter of the function name
+ # (after any "ds." prefix) rather than an enumerated file list, so
+ # newly added test files fall into a bucket automatically. dsdanger is
+ # a separate, smaller suite (steps below are gated on
+ # matrix.category == 'dsdanger') folded in here so it groups under this
+ # job's summary box instead of a standalone job.
+ matrix:
+ include:
+ - category: smk-1
+ filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)'
+ - category: smk-2
+ filter: 'smk-ds.[m-vM-V]'
+ - category: arg
+ filter: 'arg-'
+ - category: perf-1
+ filter: 'perf-ds.[a-cA-C]'
+ - category: perf-2
+ filter: 'perf-ds.[d-mD-M]'
+ - category: perf-3
+ filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)'
+ - category: misc
+ filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-'
+ - category: dsdanger
+ env:
+ PROJECT_NAME: dsBaseClient
+ DS_DRIVER: ArmadilloDriver
+ DSDANGER_TARBALL: dsDanger_6.3.4.tar.gz
+ steps:
+ - name: Checkout dsBaseClient
+ uses: actions/checkout@v5
+ with:
+ path: dsBaseClient
+
+ - uses: ./dsBaseClient/.github/actions/setup-armadillo-with-dsbase
+ with:
+ dsbase-ref: ${{ env.DSBASE_REF }}
+
+ - name: Install dsDangerClient
+ if: matrix.category == 'dsdanger'
run: |
- sleep 60
- R -q -f "molgenis_armadillo-upload_testing_datasets.R"
- working-directory: dsBaseClient/tests/testthat/data_files
+ R -q -e "
+ ref <- Sys.getenv('BRANCH_NAME')
+ ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE)
+ if (!ok) pak::pkg_install('github::datashield/dsDangerClient')"
- - name: Install dsBase to Armadillo
+ - name: Install dsDanger package on Armadillo server
+ if: matrix.category == 'dsdanger'
run: |
- curl -u admin:admin -X GET http://localhost:8080/packages
- curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@dsBase_7.0.0-permissive.tar.gz" -X POST http://localhost:8080/install-package
- sleep 60
- docker restart dsbaseclient-armadillo-1
- sleep 30
- curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase
+ curl -u admin:admin http://localhost:8080/whitelist
+ install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSDANGER_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package)
+ if [ "$install_status" != "200" ]; then
+ echo "dsDanger install request failed with HTTP status $install_status"
+ exit 1
+ fi
+
+ for i in $(seq 1 30); do
+ packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true)
+ if echo "$packages_json" | jq -e '.[] | select(.name == "dsDanger")' >/dev/null 2>&1; then
+ break
+ fi
+ sleep 10
+ done
+
+ curl -u admin:admin -X POST http://localhost:8080/whitelist/dsDanger
+ curl -u admin:admin http://localhost:8080/whitelist
working-directory: dsBaseClient
- - name: Run tests with coverage & JUnit report
+ - name: Run dsBase tests with coverage & JUnit report
+ if: matrix.category != 'dsdanger'
run: |
- mkdir -p logs
- R -q -e "devtools::reload();"
+ R -q -e "devtools::load_all();"
R -q -e '
- write.csv(
- covr::coverage_to_list(
- covr::package_coverage(
- type = c("none"),
- code = c('"'"'
- output_file <- file("test_console_output.txt");
- sink(output_file);
- sink(output_file, type = "message");
- junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml"));
- progress_rep <- testthat::ProgressReporter$new(max_failures = 999999);
- multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep));
- options("datashield.return_errors" = FALSE, "default_driver" = "ArmadilloDriver");
- testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"'
- )
- )
- ),
- "coveragelist.csv"
- )'
-
- mv coveragelist.csv logs/
- mv test_* logs/
+ cov <- covr::package_coverage(
+ type = c("none"),
+ code = c('"'"'
+ output_file <- file("test_console_output_dsbase.txt");
+ sink(output_file, split = TRUE);
+ junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml"));
+ progress_rep <- testthat::ProgressReporter$new(max_failures = 999999);
+ multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep));
+ options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
+ testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"'
+ )
+ )
+ saveRDS(cov, "coverage.rds")
+ write.csv(covr::coverage_to_list(cov), "coveragelist.csv")
+ covr::to_cobertura(cov, "cobertura.xml")' || R_EXIT=$?
+ cat test_console_output_dsbase.txt
+ n_tests=$(grep -c ' entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass."
+ R_EXIT=1
+ fi
+ n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsbase.xml | wc -l | tr -d ' ')
+ if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then
+ echo "$n_failed test failure(s)/error(s) in test_results_dsbase.xml - failing this shard."
+ R_EXIT=1
+ fi
+ exit "${R_EXIT:-0}"
working-directory: dsBaseClient
- - name: Check for JUnit errors
+ - name: Run dsDanger tests with JUnit report
+ if: matrix.category == 'dsdanger'
run: |
- issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true)
- echo "Number of testsuites with issues: $issue_count"
- sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true
- cat issues.log || true
- # continue with workflow even when some tests fail
- exit 0
- working-directory: dsBaseClient/logs
-
- - name: Write versions to file
+ R -q -e '
+ devtools::load_all(quiet = TRUE);
+ library(testthat);
+ output_file <- file("test_console_output_dsdanger.txt");
+ sink(output_file, split = TRUE);
+ junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml"));
+ progress_rep <- ProgressReporter$new(max_failures = 999999);
+ multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
+ options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
+ test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$?
+ cat test_console_output_dsdanger.txt
+ n_tests=$(grep -c ' entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass."
+ R_EXIT=1
+ fi
+ n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsdanger.xml | wc -l | tr -d ' ')
+ if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then
+ echo "$n_failed test failure(s)/error(s) in test_results_dsdanger.xml - failing this shard."
+ R_EXIT=1
+ fi
+ exit "${R_EXIT:-0}"
+ working-directory: dsBaseClient
+
+ # Written regardless of outcome so armadillo-report can tell a shard
+ # that never reported (crashed in setup, before any test XML existed)
+ # apart from one that reported 0 failures, and so the coverage step
+ # below knows how many Codecov sessions to actually expect. mkdir -p so
+ # this still succeeds even if checkout itself is what failed. Written
+ # under dsBaseClient/, alongside everything else in the same upload
+ # below, so upload-artifact doesn't widen its common-ancestor
+ # computation to the workspace root and shift every other file down an
+ # extra directory level.
+ - name: Record shard outcome
+ if: always()
run: |
- echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt
- echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt
- echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt
- working-directory: dsBaseClient/logs
+ mkdir -p dsBaseClient
+ echo "${{ job.status }}" > dsBaseClient/shard_status.txt
+
+ - name: Upload shard results
+ if: always() && matrix.category != 'dsdanger'
+ uses: actions/upload-artifact@v6
+ with:
+ name: armadillo-dsbase-${{ matrix.category }}
+ path: |
+ dsBaseClient/test_results_dsbase.xml
+ dsBaseClient/test_console_output_dsbase.txt
+ dsBaseClient/coveragelist.csv
+ dsBaseClient/coverage.rds
+ dsBaseClient/dsbase_version.txt
+ dsBaseClient/shard_status.txt
+
+ - name: Upload dsDanger results
+ if: always() && matrix.category == 'dsdanger'
+ uses: actions/upload-artifact@v6
+ with:
+ name: armadillo-dsdanger
+ path: |
+ dsBaseClient/test_results_dsdanger.xml
+ dsBaseClient/test_console_output_dsdanger.txt
+ dsBaseClient/shard_status.txt
+
+ - name: Upload coverage to Codecov
+ if: always() && matrix.category != 'dsdanger'
+ uses: codecov/codecov-action@v6
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ files: dsBaseClient/cobertura.xml
+ flags: armadillo-${{ matrix.category }}
+ fail_ci_if_error: false
+
+
+ ################################################################################
+ # Armadillo - merge all matrix entry results and publish the report.
+ ################################################################################
+ armadillo-report:
+ name: Armadillo report
+ needs: [armadillo-dsbase]
+ if: always()
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ pull-requests: write
+ steps:
+ - name: Checkout dsBaseClient
+ uses: actions/checkout@v5
+ with:
+ path: dsBaseClient
- - name: Parse results from testthat and covr
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: release
+ use-public-rspm: true
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ env:
+ PKG_INCLUDE_LINKINGTO: true
+ with:
+ working-directory: dsBaseClient
+ dependencies: 'c("Depends", "Imports", "LinkingTo")'
+ extra-packages: |
+ cran::xml2
+
+ - name: Download shard/dsdanger results
+ uses: actions/download-artifact@v7
+ with:
+ pattern: 'armadillo-*'
+ path: dsBaseClient/artifacts
+
+ - name: Merge JUnit results
run: |
- Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/
+ mkdir -p logs
+ cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt
+
+ Rscript -e '
+ xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE)
+ docs <- lapply(xml_files, xml2::read_xml)
+ root <- xml2::xml_new_root("testsuites")
+ for (doc in docs) {
+ for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s)
+ }
+ xml2::write_xml(root, "logs/test_results.xml")
+ '
working-directory: dsBaseClient
- - name: Render report
- run: |
- cd testStatus
+ - name: Upload merged results
+ uses: actions/upload-artifact@v6
+ with:
+ name: armadillo-report-results
+ path: |
+ dsBaseClient/logs/test_results.xml
+ dsBaseClient/logs/test_console_output.txt
+
+ - name: Compute coverage
+ id: coverage
+ # Codecov already computes patch (diff) coverage for this commit from
+ # the cobertura.xml uploaded by each armadillo-dsbase shard, gated at
+ # 80% in codecov.yml - that is the branch-level figure we want here.
+ # We used to poll the "codecov/patch" GitHub commit status for it,
+ # but this repo's Codecov integration never posts that status/check
+ # (confirmed via the GitHub API - only the PR comment feature is
+ # active), so we poll Codecov's own public API instead, which has
+ # the same data regardless of GitHub status-posting being enabled.
+ uses: actions/github-script@v8
+ with:
+ script: |
+ let prNumber = context.payload.pull_request?.number;
+ if (!prNumber) {
+ const branch = context.ref.replace('refs/heads/', '');
+ const prs = await github.rest.pulls.list({
+ owner: context.repo.owner, repo: context.repo.repo,
+ head: `${context.repo.owner}:${branch}`, state: 'open'
+ });
+ prNumber = prs.data[0]?.number;
+ }
+
+ // One session per armadillo-dsbase shard that uploads coverage
+ // (dsdanger is excluded - see the matrix above); Codecov's report
+ // isn't final until all of them have landed. Counted from each
+ // shard's own shard_status.txt (written in armadillo-dsbase,
+ // downloaded above) rather than hardcoded, so a shard that never
+ // got as far as generating cobertura.xml isn't waited on forever,
+ // and this doesn't need updating if the matrix is resharded.
+ const fs = require('fs');
+ const artifactsDir = 'dsBaseClient/artifacts';
+ let expectedSessions = 0;
+ if (fs.existsSync(artifactsDir)) {
+ for (const dir of fs.readdirSync(artifactsDir)) {
+ if (dir.includes('dsdanger')) continue;
+ const statusFile = `${artifactsDir}/${dir}/shard_status.txt`;
+ if (fs.existsSync(statusFile) && fs.readFileSync(statusFile, 'utf8').trim() === 'success') {
+ expectedSessions++;
+ }
+ }
+ }
+ const EXPECTED_SESSIONS = Math.max(expectedSessions, 1);
+ const THRESHOLD = 80;
+
+ let icon = '❓';
+ let text = 'no PR found to look up patch coverage for';
+ let project = 'unknown';
+ let url = `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
+
+ if (prNumber) {
+ url = `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}`;
+ icon = '❓';
+ text = 'no codecov patch data found after polling';
+ for (let i = 0; i < 18; i++) {
+ if (i > 0) await new Promise(r => setTimeout(r, 10000));
+ const res = await fetch(`https://api.codecov.io/api/v2/github/${context.repo.owner}/repos/${context.repo.repo}/pulls/${prNumber}/`);
+ if (!res.ok) continue;
+ const data = await res.json();
+
+ // Whole-project total (informational only per codecov.yml) -
+ // report it as soon as it's available, independent of the
+ // patch-readiness gate below.
+ if (typeof data.head_totals?.coverage === 'number') {
+ project = `${data.head_totals.coverage.toFixed(1)}%`;
+ }
- mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
- mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
- mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/
+ if ((data.head_totals?.sessions ?? 0) < EXPECTED_SESSIONS) continue;
- # Copy logs to new logs directory location
- cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
- cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
+ // Codecov returns patch: null, rather than a zeroed object,
+ // when the PR touches no covr-instrumented line at all.
+ const { hits, misses, partials, coverage } = data.patch ?? { hits: 0, misses: 0, partials: 0 };
+ if (hits + misses + partials === 0) {
+ icon = 'ℹ️';
+ text = 'no coverable lines changed';
+ } else {
+ icon = coverage >= THRESHOLD ? '✅' : '❌';
+ text = `${coverage.toFixed(1)}% vs ${THRESHOLD}% target`;
+ }
+ break;
+ }
+ }
- R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))'
- mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html
- cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest
+ core.setOutput('icon', icon);
+ core.setOutput('text', text);
+ core.setOutput('project', project);
+ core.setOutput('url', url);
+ - name: Compute results & write summary
+ id: results
env:
- PROJECT_NAME: ${{ env.PROJECT_NAME }}
- BRANCH_NAME: ${{ env.BRANCH_NAME }}
- WORKFLOW_ID: ${{ env.WORKFLOW_ID }}
+ ARMADILLO_DSBASE_RESULT: ${{ needs.armadillo-dsbase.result }}
+ run: |
+ Rscript -e '
+ source(".github/scripts/summarise-junit.R")
+ res <- summarise_junit("logs/test_results.xml", "Armadillo", "artifacts")
+ writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY"))
+ cat(res$summary, sep = "\n")
+
+ version <- find_dsbase_version("artifacts")
+
+ # needs.armadillo-dsbase.result is "failure" if ANY matrix shard did
+ # not succeed (even if the shards that DID upload results show 0
+ # failures) - a shard that never reported must not look like a pass.
+ shard_ok <- Sys.getenv("ARMADILLO_DSBASE_RESULT") == "success"
+ ok <- res$ok && shard_ok
+ # Only a fallback: the usual case (a shard crashed/reported 0
+ # tests) is already named above via res$shard_problems - this
+ # covers the rare gap where a shard job failed without leaving
+ # any trace summarise_junit() could identify.
+ if (!shard_ok && length(res$shard_problems) == 0) {
+ message("One or more Armadillo dsbase/dsdanger matrix entries did not succeed, but none could be identified from their uploaded results.")
+ }
+
+ out <- Sys.getenv("GITHUB_OUTPUT")
+ cat(
+ sprintf("ok=%s\n", tolower(ok)),
+ sprintf("tally=%s\n", res$tally),
+ sprintf("version=%s\n", version),
+ file = out, append = TRUE, sep = ""
+ )
- - name: Upload test logs
- uses: actions/upload-artifact@v4
+ if (!ok) message("Armadillo tests failed.")
+ quit(save = "no", status = if (ok) 0 else 1)
+ '
+ working-directory: dsBaseClient
+
+ - name: Post PR comment
+ if: always()
+ uses: actions/github-script@v8
with:
- name: dsbaseclient-logs
- path: testStatus/new
+ script: |
+ // steps.results.outputs.* come back empty (not "true"/"false") if
+ // that step never got far enough to write them - e.g. it errored
+ // or was skipped outright because an earlier step failed. Fall
+ // back to something informative rather than a blank tally.
+ const ok = '${{ steps.results.outputs.ok }}' === 'true';
+ const tally = '${{ steps.results.outputs.tally }}' || 'error - see log';
+ const version = '${{ steps.results.outputs.version }}' || 'unknown';
+ const coverageIcon = '${{ steps.coverage.outputs.icon }}' || '❓';
+ const coverageText = '${{ steps.coverage.outputs.text }}' || 'error - see log';
+ const coverageProject = '${{ steps.coverage.outputs.project }}' || 'unknown';
+ // #summary- scrolls straight to this job's summary card
+ // (the pass/fail tally written via GITHUB_STEP_SUMMARY) instead
+ // of just the top of the run page.
+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}#summary-${{ job.check_run_id }}`;
+ const codecovUrl = '${{ steps.coverage.outputs.url }}' || `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
+
+ const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js');
+ await postCiComment({ github, context, updates: {
+ 'row:tests-armadillo': `
`,
+ 'log:lint': `_pending_`
+ }});
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: release
+ use-public-rspm: true
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ env:
+ PKG_INCLUDE_LINKINGTO: true
+ with:
+ dependencies: 'c("Depends", "Imports", "LinkingTo")'
+ extra-packages: |
+ local::.
+ cran::lintr
+ cran::jsonlite
+
+ - name: Get PR diff
+ if: github.event_name == 'pull_request'
+ id: changed
+ run: |
+ # Three-dot diff (against the merge-base) rather than two-dot: when
+ # the base branch has diverged and gained its own independent
+ # commits, a plain two-dot diff shows those too, not just what this
+ # PR actually added - matching what GitHub's own "Files changed"
+ # tab and Advanced Security baseline compare against.
+ patch=$(git diff --unified=0 "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" -- '*.R' '*.Rmd')
+ echo "patch<> "$GITHUB_ENV"
+ echo "$patch" >> "$GITHUB_ENV"
+ echo "DIFF_PATCH_EOF" >> "$GITHUB_ENV"
+
+ - name: Lint
+ id: lint
+ env:
+ IS_PR: ${{ github.event_name == 'pull_request' }}
+ DIFF_PATCH: ${{ env.patch }}
+ run: |
+ Rscript -e '
+ al <- lintr::available_linters()
+ keep <- vapply(al$tags, function(t) any(t %in% c("correctness", "common_mistakes", "robustness")), logical(1))
+ sel <- al$linter[keep]
+ linter_funs <- lapply(sel, function(n) get(n, envir = asNamespace("lintr"))())
+ names(linter_funs) <- sel
+ lints <- lintr::lint_package(linters = linter_funs)
+ cat("n lints:", length(lints), "\n")
+ print(lints) # emits ::warning file=...,line=...:: annotations (auto-detects GitHub Actions)
+ lintr::sarif_output(lints, "lintr_results.sarif")
+
+ is_pr <- Sys.getenv("IS_PR") == "true"
+
+ # A finding only counts as "new" if the PR diff actually added or
+ # changed that exact line - matching a file merely being touched
+ # elsewhere (the old approach) blamed PRs for pre-existing issues
+ # anywhere in a file they only partly edited.
+ parse_added_lines <- function(patch) {
+ added <- new.env()
+ cur_file <- NULL
+ for (ln in strsplit(patch, "\n")[[1]]) {
+ if (startsWith(ln, "+++ ")) {
+ path <- sub("^\\+\\+\\+ b/", "", ln)
+ cur_file <- if (identical(path, "/dev/null")) NULL else path
+ } else if (!is.null(cur_file) && startsWith(ln, "@@")) {
+ m <- regmatches(ln, regexpr("\\+[0-9]+(,[0-9]+)?", ln))
+ if (length(m) == 1 && nzchar(m)) {
+ parts <- strsplit(sub("^\\+", "", m), ",")[[1]]
+ new_start <- as.integer(parts[1])
+ new_count <- if (length(parts) > 1) as.integer(parts[2]) else 1L
+ if (new_count > 0) {
+ existing <- if (is.null(added[[cur_file]])) integer(0) else added[[cur_file]]
+ added[[cur_file]] <- c(existing, seq(new_start, length.out = new_count))
+ }
+ }
+ }
+ }
+ added
+ }
+
+ added_lines <- if (is_pr) parse_added_lines(Sys.getenv("DIFF_PATCH")) else new.env()
+ is_new <- vapply(lints, function(l) {
+ lines <- added_lines[[l$filename]]
+ !is.null(lines) && l$line_number %in% lines
+ }, logical(1))
+ n_new <- sum(is_new)
+ n_total <- length(lints)
+
+ cat(
+ sprintf("n_new=%d\n", n_new),
+ sprintf("n_total=%d\n", n_total),
+ file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE, sep = ""
+ )
+
+ if (n_new > 0) {
+ message(sprintf("Lint found %d issue(s) in files changed by this PR - see the annotations above or the SARIF upload for details.", n_new))
+ }
+ quit(save = "no", status = if (n_new > 0) 1 else 0)
+ '
+
+ - name: Upload lint results
+ if: always()
+ uses: github/codeql-action/upload-sarif@v4
+ with:
+ sarif_file: lintr_results.sarif
+
+ - name: Post PR comment
+ if: always() && github.event_name == 'pull_request'
+ uses: actions/github-script@v8
+ with:
+ script: |
+ const nNew = parseInt('${{ steps.lint.outputs.n_new }}' || '0', 10);
+ const nTotal = '${{ steps.lint.outputs.n_total }}' || 'unknown';
+ const ok = nNew === 0;
+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+ const postCiComment = require('${{ github.workspace }}/.github/scripts/post-ci-comment.js');
+ await postCiComment({ github, context, updates: {
+ 'row:lint': `
Code quality
${ok ? '✅ 0 new findings' : `❌ ${nNew} new finding${nNew === 1 ? '' : 's'}`} (package total: ${nTotal})
`,
+ 'log:lint': `Code quality`
+ }});
diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml
index bfc9f4db3..d316baaf3 100644
--- a/.github/workflows/pkgdown.yaml
+++ b/.github/workflows/pkgdown.yaml
@@ -23,7 +23,7 @@ jobs:
permissions:
contents: write
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- uses: r-lib/actions/setup-pandoc@v2
@@ -42,7 +42,7 @@ jobs:
- name: Deploy to GitHub pages 🚀
if: github.event_name != 'pull_request'
- uses: JamesIves/github-pages-deploy-action@v4.5.0
+ uses: JamesIves/github-pages-deploy-action@v4.9.0
with:
clean: false
branch: gh-pages
diff --git a/.gitignore b/.gitignore
index 60d56797e..93decc0f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ azure-pipelines.Rout
tests/testthat/connection_to_datasets/local_settings.csv
tests/docker/armadillo/standard/logs/
tests/docker/armadillo/standard/data/
+lintr_results.sarif
diff --git a/DESCRIPTION b/DESCRIPTION
index 6e41033db..dd838b8c2 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -64,8 +64,8 @@ Authors@R: c(person(given = "Paul",
affiliation = "Genomics Coordination Centre, UMCG, Netherlands")))
License: GPL-3
Depends:
- R (>= 4.0.0),
- DSI (>= 1.7.1)
+ R (>= 4.1.0),
+ DSI (>= 1.8.0)
Imports:
cli,
fields,
@@ -88,6 +88,6 @@ Suggests:
DSOpal,
DSMolgenisArmadillo,
DSLite
-RoxygenNote: 8.0.0
Encoding: UTF-8
Language: en-GB
+Config/roxygen2/version: 8.1.0
diff --git a/NAMESPACE b/NAMESPACE
index bd539a118..88498755c 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -121,7 +121,9 @@ import(DSI)
import(data.table)
importFrom(DSI,datashield.connections_find)
importFrom(cli,cli_abort)
-importFrom(stats,as.formula)
-importFrom(stats,na.omit)
-importFrom(stats,ts)
-importFrom(stats,weighted.mean)
+importFrom(stats,
+ as.formula,
+ na.omit,
+ ts,
+ weighted.mean
+)
diff --git a/R/ds.asFactor.R b/R/ds.asFactor.R
index 8e5fbd090..e6b6e7ce2 100644
--- a/R/ds.asFactor.R
+++ b/R/ds.asFactor.R
@@ -133,10 +133,8 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.asFactor} returns the unique levels of the converted
-#' variable in ascending order and a validity
-#' message with the name of the created object on the client-side and
-#' the output matrix or vector in the server-side.
+#' @return \code{ds.asFactor} returns the unique levels of the converted
+#' variable in ascending order. The output matrix or vector is written to the server-side.
#'
#' @examples
#' \dontrun{
@@ -185,19 +183,12 @@
#' datashield.logout(connections)
#' }
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.asFactor <- function(input.var.name=NULL, newobj.name=NULL, forced.factor.levels=NULL, fixed.dummy.vars=FALSE,
baseline.level=1, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of the column that holds the input variable
if(is.null(input.var.name)){
@@ -248,58 +239,7 @@ ds.asFactor <- function(input.var.name=NULL, newobj.name=NULL, forced.factor.lev
calltext2 <- call("asFactorDS2", input.var.name, all.unique.levels.transmit, fixed.dummy.vars, baseline.level)
DSI::datashield.assign(datasources, newobj.name, calltext2)
-##########################################################################################################
-#MODULE 5: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj.name #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("Data object <", test.obj.name, "> correctly created in all specified data sources") #
- #
- return(list(all.unique.levels=all.unique.levels,return.message=return.message)) #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources")#
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message<-list(return.message.1,return.message.2) #
- #
- return.info<-object.info #
- #
-return(list(all.unique.levels=all.unique.levels,return.info=return.info,return.message=return.message)) #
- #
- } #
-#END OF MODULE 5 #
-##########################################################################################################
-
+ return(list(all.unique.levels=all.unique.levels))
}
#ds.asFactor
diff --git a/R/ds.asFactorSimple.R b/R/ds.asFactorSimple.R
index 313f7b408..fc092d352 100644
--- a/R/ds.asFactorSimple.R
+++ b/R/ds.asFactorSimple.R
@@ -17,23 +17,14 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return an output vector of class factor to the serverside. In addition, returns a validity
-#' message with the name of the created object on the client-side and if creation fails an
-#' error message which can be viewed using datashield.errors().
+#' @return an output vector of class factor written to the serverside.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.asFactorSimple <- function(input.var.name=NULL, newobj.name=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of the column that holds the input variable
if(is.null(input.var.name)){
@@ -55,58 +46,5 @@ ds.asFactorSimple <- function(input.var.name=NULL, newobj.name=NULL, datasources
calltext0 <- call("asFactorSimpleDS", input.var.name)
DSI::datashield.assign(datasources, newobj.name, calltext0)
-##########################################################################################################
-#MODULE 5: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj.name #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("Data object <", test.obj.name, "> correctly created in all specified data sources") #
- #
- return(list(return.message=return.message)) #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources")#
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message<-list(return.message.1,return.message.2) #
- #
- return.info<-object.info #
- #
-return(list(return.info=return.info,return.message=return.message)) #
- #
- } #
-#END OF MODULE 5 #
-##########################################################################################################
-
-
}
#ds.asFactorSimple
diff --git a/R/ds.changeRefGroup.R b/R/ds.changeRefGroup.R
index 4bd5080ae..a06ac1435 100644
--- a/R/ds.changeRefGroup.R
+++ b/R/ds.changeRefGroup.R
@@ -28,6 +28,7 @@
#' @return \code{ds.changeRefGroup} returns a new vector with the specified level as a reference
#' which is written to the server-side.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @seealso \code{\link{ds.cbind}} Combines objects column-wise.
#' @seealso \code{\link{ds.levels}} to obtain the levels (categories) of a vector of type factor.
#' @seealso \code{\link{ds.colnames}} to obtain the column names of a matrix or a data frame
@@ -109,15 +110,7 @@
#' @export
ds.changeRefGroup <- function(x=NULL, ref=NULL, newobj=NULL, reorderByRef=FALSE, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
if(is.null(x)){
stop("Please provide the name of a vector of type factor!", call.=FALSE)
@@ -132,26 +125,12 @@ ds.changeRefGroup <- function(x=NULL, ref=NULL, newobj=NULL, reorderByRef=FALSE,
newobj <- "changerefgroup.newobj"
}
- # check if the input object is defined in all the studies
- isDefined(datasources, x)
-
- # call the internal function that checks the input object is of the same class in all studies.
- typ <- checkClass(datasources, x)
-
- # if input vector is not a factor stop
- if(!('factor' %in% typ)){
- stop("The input vector must be a factor!", call.=FALSE)
- }
-
if(reorderByRef){
warning("'reorderByRef' is set to TRUE. Please read the documentation for possible consequences!", call.=FALSE)
}
# call the server side function that will recode the levels
- cally <- paste0('changeRefGroupDS(', x, ",'", ref, "',", reorderByRef,")")
- DSI::datashield.assign(datasources, newobj, as.symbol(cally))
-
- # check that the new object has been created and display a message accordingly
- finalcheck <- isAssigned(datasources, newobj)
+ calltext <- call("changeRefGroupDS", x, ref, reorderByRef)
+ DSI::datashield.assign(datasources, newobj, calltext)
}
diff --git a/R/ds.dmtC2S.R b/R/ds.dmtC2S.R
index 085d198fb..3c032b711 100644
--- a/R/ds.dmtC2S.R
+++ b/R/ds.dmtC2S.R
@@ -41,20 +41,13 @@
#' @return the object specified by the argument (or default name "dmt.copied.C2S")
#' which is written as a data.frame/matrix/tibble to the serverside.
#' @author Paul Burton for DataSHIELD Development Team - 3rd June, 2021
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.dmtC2S <- function(dfdata=NA, newobj=NULL, datasources=NULL){
- # if no opal login details are provided look for 'opal' objects in the environment
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
-
+ datasources <- .set_datasources(datasources)
+
# check if a value has been provided for dfdata
if(is.null(dfdata)){
return("Error: dfdata must be a character string, a numeric vector or a scalar")
diff --git a/R/ds.gamlss.R b/R/ds.gamlss.R
index 6a7622c76..4a0d040ed 100644
--- a/R/ds.gamlss.R
+++ b/R/ds.gamlss.R
@@ -72,6 +72,7 @@
#' residuals (the normalised quantile residuals of the model) are not disclosed to
#' the client-side.
#' @author Demetris Avraam for DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.gamlss <- function(formula = NULL, sigma.formula = '~1', nu.formula = '~1',
@@ -81,16 +82,8 @@ ds.gamlss <- function(formula = NULL, sigma.formula = '~1', nu.formula = '~1',
i.control = c(0.001, 50, 30, 0.001), centiles = FALSE,
xvar = NULL, newobj = NULL, datasources = NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- DSI::datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
-
+ datasources <- .set_datasources(datasources)
+
# verify that 'formula' was set
if(is.null(formula)){
stop(" Please provide a valid formula!", call.=FALSE)
diff --git a/R/ds.glm.R b/R/ds.glm.R
index 8b1dbceb7..24488e226 100644
--- a/R/ds.glm.R
+++ b/R/ds.glm.R
@@ -211,6 +211,7 @@
#' and the natural scale after exponentiation (rates and rate ratios).
#'
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#' @examples
#' \dontrun{
@@ -324,15 +325,7 @@
ds.glm <- function(formula=NULL, data=NULL, family=NULL, offset=NULL, weights=NULL, checks=FALSE, maxit=20, CI=0.95,
viewIter=FALSE, viewVarCov=FALSE, viewCor=FALSE, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# verify that 'formula' was set
if(is.null(formula)){
@@ -356,11 +349,6 @@ ds.glm <- function(formula=NULL, data=NULL, family=NULL, offset=NULL, weights=NU
stop(" Please provide a valid 'family' argument!", call.=FALSE)
}
- # if the argument 'data' is set, check that the data frame is defined (i.e. exists) on the server site
- if(!(is.null(data))){
- defined <- isDefined(datasources, data)
- }
-
# beginning of optional checks - the process stops if any of these checks fails #
if(checks){
message(" -- Verifying the variables in the model")
diff --git a/R/ds.glmPredict.R b/R/ds.glmPredict.R
index 96dfc792c..e881bb541 100644
--- a/R/ds.glmPredict.R
+++ b/R/ds.glmPredict.R
@@ -121,29 +121,19 @@
#' summary statistics for each column in fit and se.fit matrices which each have k columns
#' if k terms are being summarised.
#' @author Paul Burton, for DataSHIELD Development Team 13/08/20
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.glmPredict <- function(glmname = NULL, newdataname = NULL, output.type = "response",
se.fit = FALSE, dispersion = NULL, terms = NULL,
na.action = "na.pass", newobj = NULL, datasources = NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check that is set
if(is.null(glmname)){
stop(" is not set, please specify it as a character string containing the name of a valid glm class object on the serverside", call.=FALSE)
}
-
- # check if the glm object is defined in all the studies
- isDefined(datasources, glmname)
# check that is correctly set
if((output.type!="link") && (output.type!="response") && (output.type!="terms")){
@@ -163,85 +153,6 @@ ds.glmPredict <- function(glmname = NULL, newdataname = NULL, output.type = "res
calltext <- call("glmPredictDS.as", glmname, newdataname, output.type, se.fit, dispersion, terms, na.action)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(object.info[[j]]$test.obj.class=="ABSENT"){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
-# print(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
-# print(list(is.object.created=return.message,validity.check=validity.check, #
-# studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
calltext<-call("glmPredictDS.ag", glmname, newdataname, output.type,
se.fit, dispersion, terms, na.action)
diff --git a/R/ds.glmSLMA.R b/R/ds.glmSLMA.R
index 3c9d0edb5..ef366e21f 100644
--- a/R/ds.glmSLMA.R
+++ b/R/ds.glmSLMA.R
@@ -261,12 +261,8 @@
#' argument combine.with.metafor is set to TRUE. Otherwise, users can take
#' the \code{betamatrix.valid} and \code{sematrix.valid} matrices and enter
#' them into their meta-analysis package of choice.
-#' @return \code{is.object.created} and \code{validity.check} are standard
-#' items returned by an assign function when the designated newobj appears to have
-#' been successfully created on the serverside at each study. This output is
-#' produced specifically by the assign function \code{glmSLMADS.assign} that writes
-#' out the glm object on the serverside
#' @author Paul Burton, for DataSHIELD Development Team 07/07/20
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -377,16 +373,7 @@
ds.glmSLMA<-function(formula=NULL, family=NULL, offset=NULL, weights=NULL, combine.with.metafor=TRUE,
newobj=NULL,dataName=NULL,checks=FALSE, maxit=30, notify.of.progress=FALSE, datasources=NULL) {
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
-
+ datasources <- .set_datasources(datasources)
# verify that 'formula' was set
if(is.null(formula)){
@@ -454,11 +441,6 @@ ds.glmSLMA<-function(formula=NULL, family=NULL, offset=NULL, weights=NULL, combi
if(paste(strsplit(family,split=" ")[[1]],collapse="")=="gamma(link=log)")
{family<-"Gamma.link.log"}
- # if the argument 'dataName' is set, check that the data frame is defined (i.e. exists) on the server site
- if(!(is.null(dataName))){
- defined <- isDefined(datasources, dataName)
- }
-
# beginning of optional checks - the process stops if any of these checks fails #
if(checks){
message(" -- Verifying the variables in the model")
@@ -842,94 +824,10 @@ return(list(output.summary=output.summary))
}
-#final.outlist<-(list(output.summary=output.summary, num.valid.studies=num.valid.studies,betamatrix.all=betamatrix.all,sematrix.all=sematrix.all, betamatrix.valid=betamatrix.valid,sematrix.valid=sematrix.valid,
-# SLMA.pooled.ests.matrix=SLMA.pooled.ests.matrix))
-
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-datashield.aggregate(datasources, calltext)
- #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if("ABSENT" %in% object.info[[j]]$test.obj.class){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(output.summary=output.summary, num.valid.studies=num.valid.studies, #
- betamatrix.all=betamatrix.all, #
- sematrix.all=sematrix.all, betamatrix.valid=betamatrix.valid,sematrix.valid=sematrix.valid, #
- SLMA.pooled.ests.matrix=SLMA.pooled.ests.matrix, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORRECTLY MODULE #
-#############################################################################################################
+ return(list(output.summary=output.summary, num.valid.studies=num.valid.studies,
+ betamatrix.all=betamatrix.all,
+ sematrix.all=sematrix.all, betamatrix.valid=betamatrix.valid, sematrix.valid=sematrix.valid,
+ SLMA.pooled.ests.matrix=SLMA.pooled.ests.matrix))
}
diff --git a/R/ds.glmSummary.R b/R/ds.glmSummary.R
index 9fc259c5b..5a80b7fce 100644
--- a/R/ds.glmSummary.R
+++ b/R/ds.glmSummary.R
@@ -74,27 +74,17 @@
#' For further information see help for glm and summary(glm) in native R
#' and for ds.glmSLMA in DataSHIELD.
#' @author Paul Burton, for DataSHIELD Development Team 17/07/20
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.glmSummary <- function(x.name, newobj=NULL, datasources=NULL) {
- # if no connections are specified look for connection objects in the environment
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if a value has been provided for x
if(is.null(x.name)||!is.character(x.name)){
stop("Error: x.name must denote a character string naming the glm object on the serverside to be summarised", call.=FALSE)
}
-
- # check if the input object is defined in all the studies
- isDefined(datasources, x.name)
# create a name by default if the user did not provide a name for the new object
if (is.null(newobj)) {
@@ -109,83 +99,6 @@ ds.glmSummary <- function(x.name, newobj=NULL, datasources=NULL) {
# PREPARE AND CALL THE SECOND ASSIGN FUNCTION TO PREPARE AN ABBREVIATED
# summary_glm OBJECT ON THE SERVERSIDE THAT CAN SAFELY BE RETURNED TO CLIENT
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
-# if(no.errors){ #
-# message("\n\nCREATE ASSIGN OBJECT\n") #
-# #
-# validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
-# print(list(is.object.created=return.message,validity.check=validity.check)) #
-# } #
- #
- if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- message(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
# PREPARE AND CALL THE SECOND SERVERSIDE (AGGREGATE) FUNCTION TO PREPARE AN ABBREVIATED
diff --git a/R/ds.glmerSLMA.R b/R/ds.glmerSLMA.R
index b996707ef..0665b24e5 100644
--- a/R/ds.glmerSLMA.R
+++ b/R/ds.glmerSLMA.R
@@ -241,6 +241,7 @@
#'
#'
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.glmerSLMA <- function(formula=NULL, offset=NULL, weights=NULL, combine.with.metafor=TRUE, dataName=NULL,
@@ -249,15 +250,7 @@ ds.glmerSLMA <- function(formula=NULL, offset=NULL, weights=NULL, combine.with.m
start_theta = NULL, start_fixef = NULL, notify.of.progress=FALSE,
assign=FALSE, newobj=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# verify that 'formula' was set
if(is.null(formula)){
@@ -282,11 +275,6 @@ ds.glmerSLMA <- function(formula=NULL, offset=NULL, weights=NULL, combine.with.m
stop(" Please provide a valid 'family' argument!", call.=FALSE)
}
- # if the argument 'dataName' is set, check that the data frame is defined (i.e. exists) on the server site
- if(!(is.null(dataName))){
- defined <- isDefined(datasources, dataName)
- }
-
# beginning of optional checks - the process stops if any of these checks fails #
if(checks){
message(" -- Verifying the variables in the model")
diff --git a/R/ds.lmerSLMA.R b/R/ds.lmerSLMA.R
index 8b7c69b2c..23192f7a8 100644
--- a/R/ds.lmerSLMA.R
+++ b/R/ds.lmerSLMA.R
@@ -159,6 +159,7 @@
#' @return \code{convergence.error.message}: reports for each study whether the model converged.
#' If it did not some information about the reason for this is reported.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -205,15 +206,7 @@ ds.lmerSLMA <- function(formula=NULL, offset=NULL, weights=NULL, combine.with.me
control_value = NULL, optimizer = NULL, verbose = 0, notify.of.progress=FALSE,
assign=FALSE, newobj=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# verify that 'formula' was set
if(is.null(formula)){
@@ -236,11 +229,6 @@ ds.lmerSLMA <- function(formula=NULL, offset=NULL, weights=NULL, combine.with.me
# set family to gaussian
family <- 'gaussian'
- # if the argument 'dataName' is set, check that the data frame is defined (i.e. exists) on the server site
- if(!(is.null(dataName))){
- defined <- isDefined(datasources, dataName)
- }
-
# beginning of optional checks - the process stops if any of these checks fails #
if(checks){
message(" -- Verifying the variables in the model")
diff --git a/R/ds.matrix.R b/R/ds.matrix.R
index 69ad3a728..5a0ed3150 100644
--- a/R/ds.matrix.R
+++ b/R/ds.matrix.R
@@ -48,11 +48,9 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrix} returns the created matrix which is written on the server-side.
-#' In addition, two validity messages are returned
-#' indicating whether the new matrix has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.matrix} returns the created matrix which is written on the server-side.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -147,15 +145,7 @@
ds.matrix <- function(mdata = NA, from="clientside.scalar", nrows.scalar=NULL, ncols.scalar=NULL, byrow = FALSE,
dimnames = NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if a value has been provided for mdata
if(is.null(mdata)){
@@ -208,85 +198,5 @@ ds.matrix <- function(mdata = NA, from="clientside.scalar", nrows.scalar=NULL, n
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrix
diff --git a/R/ds.matrixDet.R b/R/ds.matrixDet.R
index 5fcd81a53..2f4d49606 100644
--- a/R/ds.matrixDet.R
+++ b/R/ds.matrixDet.R
@@ -16,12 +16,10 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrixDet} returns the determinant of an existing matrix on the server-side.
-#' The created new object is stored on the server-side.
-#' Also, two validity messages are returned
-#' indicating whether the matrix has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.matrixDet} returns the determinant of an existing matrix on the server-side.
+#' The created new object is stored on the server-side.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -83,23 +81,12 @@
#'
ds.matrixDet<-function(M1=NULL, newobj=NULL, logarithm=FALSE, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
return("Error: Please provide the name of the matrix representing M1")
}
-
- # check if the input object is defined in all the studies
- isDefined(datasources, M1)
# if no value or invalid value specified for logarithm, then specify a default
if(is.null(logarithm)){
@@ -119,85 +106,5 @@ ds.matrixDet<-function(M1=NULL, newobj=NULL, logarithm=FALSE, datasources=NULL){
calltext <- call("matrixDetDS2", M1, logarithm)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORRECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixDet
diff --git a/R/ds.matrixDet.report.R b/R/ds.matrixDet.report.R
index be21aaa72..4d914d5c3 100644
--- a/R/ds.matrixDet.report.R
+++ b/R/ds.matrixDet.report.R
@@ -18,6 +18,7 @@
#' @return \code{ds.matrixDet.report} returns to the client-side
#' the determinant of a matrix that is stored on the server-side.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -76,15 +77,7 @@
#'
ds.matrixDet.report<-function(M1=NULL, logarithm=FALSE, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
diff --git a/R/ds.matrixDiag.R b/R/ds.matrixDiag.R
index 1ea1341a4..7b78d9bac 100644
--- a/R/ds.matrixDiag.R
+++ b/R/ds.matrixDiag.R
@@ -56,11 +56,9 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrixDiag} returns to the server-side the square matrix diagonal.
-#' Also, two validity messages are returned
-#' indicating whether the new object has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.matrixDiag} returns to the server-side the square matrix diagonal.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -177,15 +175,7 @@
#'
ds.matrixDiag<-function(x1=NULL, aim=NULL, nrows.scalar=NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if a value has been provided for x1
if(is.null(x1)){
@@ -235,85 +225,5 @@ ds.matrixDiag<-function(x1=NULL, aim=NULL, nrows.scalar=NULL, newobj=NULL, datas
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixDiag
diff --git a/R/ds.matrixDimnames.R b/R/ds.matrixDimnames.R
index 6f4a37ead..be7cb1694 100644
--- a/R/ds.matrixDimnames.R
+++ b/R/ds.matrixDimnames.R
@@ -15,11 +15,9 @@
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
#' @return \code{ds.matrixDimnames} returns to the server-side
-#' the matrix with specified row and column names.
-#' Also, two validity messages are returned to the client-side
-#' indicating the new object that has been created in each data source and if so whether
-#' it is in a valid form.
+#' the matrix with specified row and column names.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -84,15 +82,8 @@
#' @export
ds.matrixDimnames<-function(M1=NULL, dimnames=NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
@@ -110,85 +101,5 @@ ds.matrixDimnames<-function(M1=NULL, dimnames=NULL, newobj=NULL, datasources=NUL
calltext <- call("matrixDimnamesDS", M1, dimnames)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixDimnames
diff --git a/R/ds.matrixInvert.R b/R/ds.matrixInvert.R
index 8cc3c447a..d92591c41 100644
--- a/R/ds.matrixInvert.R
+++ b/R/ds.matrixInvert.R
@@ -12,11 +12,9 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrixInvert} returns to the server-side the inverts square matrix.
-#' Also, two validity messages are returned to the client-side
-#' indicating whether the new object has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.matrixInvert} returns to the server-side the inverts square matrix.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -81,15 +79,7 @@
#'
ds.matrixInvert<-function(M1=NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
@@ -105,85 +95,5 @@ ds.matrixInvert<-function(M1=NULL, newobj=NULL, datasources=NULL){
calltext <- call("matrixInvertDS", M1)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixInvert
diff --git a/R/ds.matrixMult.R b/R/ds.matrixMult.R
index cf5349fe0..35e9d7111 100644
--- a/R/ds.matrixMult.R
+++ b/R/ds.matrixMult.R
@@ -15,12 +15,10 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrixMult} returns to the server-side
+#' @return \code{ds.matrixMult} returns to the server-side
#' the result of the two matrix multiplication.
-#' Also, two validity messages are returned to the client-side
-#' indicating whether the new object has been created in each data source and if so whether
-#' it is in a valid form.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#'
#' @examples
#' \dontrun{
@@ -96,15 +94,7 @@
#'
ds.matrixMult<-function(M1=NULL, M2=NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
@@ -129,85 +119,5 @@ ds.matrixMult<-function(M1=NULL, M2=NULL, newobj=NULL, datasources=NULL){
calltext <- call("matrixMultDS", M1, M2)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixMult
diff --git a/R/ds.matrixTranspose.R b/R/ds.matrixTranspose.R
index bbd73a1a8..4de06471a 100644
--- a/R/ds.matrixTranspose.R
+++ b/R/ds.matrixTranspose.R
@@ -15,11 +15,9 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.matrixTranspose} returns to the server-side the transpose matrix.
-#' Also, two validity messages are returned to the client-side
-#' indicating whether the new object has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.matrixTranspose} returns to the server-side the transpose matrix.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -84,15 +82,7 @@
#'
ds.matrixTranspose<-function(M1=NULL, newobj=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if user has provided the name of matrix representing M1
if(is.null(M1)){
@@ -108,85 +98,5 @@ ds.matrixTranspose<-function(M1=NULL, newobj=NULL, datasources=NULL){
calltext <- call("matrixTransposeDS", M1)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.matrixTranspose
diff --git a/R/ds.mice.R b/R/ds.mice.R
index bcb473a4a..38e271572 100644
--- a/R/ds.mice.R
+++ b/R/ds.mice.R
@@ -43,29 +43,19 @@
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
#' @return a list with three elements: the method, the predictorMatrix and the post.
#' @author Demetris Avraam for DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.mice <- function(data=NULL, m=5, maxit=5, method=NULL, predictorMatrix=NULL, post=NULL,
seed=NA, newobj_mids=NULL, newobj_df=NULL, datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- DSI::datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
-
+ datasources <- .set_datasources(datasources)
+
# verify that 'data' was set
if(is.null(data)){
stop("Please provide the name of the dataframe or matrix that contains the incomplete data!", call.=FALSE)
}
-
- # check if the 'data' are defined in all the studies
- defined.data <- isDefined(datasources, data)
-
+
if(!is.null(method)){
method <- paste0(as.character(method), collapse=",")
}
diff --git a/R/ds.rBinom.R b/R/ds.rBinom.R
index ec8b4f880..1759db7d4 100644
--- a/R/ds.rBinom.R
+++ b/R/ds.rBinom.R
@@ -31,9 +31,9 @@
#' Server functions called: \code{rBinomDS} and \code{setSeedDS}.
#' @param samp.size an integer value or an integer vector that defines the length of
#' the random numeric vector to be created in each source.
-#' @param size a positive integer that specifies the number of Bernoulli trials.
+#' @param size a positive integer that specifies the number of Bernoulli trials. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param prob a numeric scalar value or vector in range 0 > prob > 1 which specifies the
-#' probability of a positive response (i.e. 1 rather than 0).
+#' probability of a positive response (i.e. 1 rather than 0). A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param newobj a character string that provides the name for the output variable
#' that is stored on the data servers. Default \code{rbinom.newobj}.
#' @param seed.as.integer an integer or a NULL value which provides the
@@ -44,13 +44,12 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.rBinom} returns random number vectors
-#' with a Binomial distribution for each study,
-#' taking into account the values specified in each parameter of the function.
-#' The output vector is written to the server-side.
-#' If requested, it also returned to the client-side the full 626 lengths
-#' random seed vector generated in each source
-#' (see info for the argument \code{return.full.seed.as.set}).
+#' @return \code{ds.rBinom} writes a random number vector with a Binomial distribution
+#' to the server-side in each study and returns a list to the client-side containing
+#' \code{integer.seed.as.set.by.source} (the trigger seed set in each source),
+#' \code{random.vector.length.by.source} (the length of the vector created in each source)
+#' and, if \code{return.full.seed.as.set} is TRUE, \code{full.seed.as.set}
+#' (the full 626 length random seed vector generated in each source).
#'
#' @examples
#' \dontrun{
@@ -83,7 +82,7 @@
#'
#' #Generating the vectors in the Opal servers
#' ds.rBinom(samp.size=c(13,20,25), #the length of the vector created in each source is different
-#' size=as.character(c(10,23,5)), #Bernoulli trials change in each source
+#' size=c(10,23,5), #Bernoulli trials change in each source
#' prob=c(0.6,0.1,0.5), #Probability changes in each source
#' newobj="Binom.dist",
#' seed.as.integer=45,
@@ -103,19 +102,11 @@
#' datashield.logout(connections)
#' }
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.rBinom<-function(samp.size=1,size=0,prob=1, newobj=NULL, seed.as.integer=NULL, return.full.seed.as.set=FALSE, datasources=NULL){
-##################################################################################
-# look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# create a name by default if user did not provide a name for the new variable
if(is.null(newobj)){
@@ -162,9 +153,13 @@ mess2<-("ERROR: appropriate values must be set for samp.size, size, prob, and ne
return(mess2)
}
+numsources<-length(datasources)
+size<-.expand_to_studies(size, "size", numsources)
+prob<-.expand_to_studies(prob, "prob", numsources)
+
size.valid<-1
if(is.numeric(size)){
- if(size<=0){
+ if(any(size<=0)){
size.valid<-0
}
}
@@ -176,7 +171,7 @@ return(mess3)
prob.valid<-1
if(is.numeric(prob)){
- if(prob<=0||prob>=1.0){
+ if(any(prob<=0|prob>=1.0)){
prob.valid<-0
}
}
@@ -217,8 +212,7 @@ if(seed.as.text=="NULL"){
message("NO SEED SET IN STUDY",study.id,"\n\n")
} else {
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
- ssDS.obj[[study.id]] <- DSI::datashield.aggregate(datasources[study.id], as.symbol(calltext))
+ ssDS.obj[[study.id]] <- datashield.aggregate(datasources[study.id], call("setSeedDS", seedtext=seed.as.text))
}
}
message("\n\n")
@@ -235,104 +229,15 @@ samp.size<-rep(samp.size,numsources)
}
for(k in 1:numsources){
+ datashield.assign(datasources[k], newobj, call("rBinomDS", samp.size[k], size=size[k], prob=prob[k]))
+}
-toAssign<-paste0("rBinomDS(",samp.size[k],",",size, ",", prob, ")")
-
-
- if(is.null(toAssign)){
- stop("Please give the name of object to assign or an expression to evaluate and assign.!\n", call.=FALSE)
- }
-
- # now do the business
-
- DSI::datashield.assign(datasources[k], newobj, as.symbol(toAssign))
- }
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors && !return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
- if(no.errors && return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(full.seed.as.set=ssDS.obj, #
- integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
+if(return.full.seed.as.set){
+return(list(full.seed.as.set=ssDS.obj,
+ integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
+}
+return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
}
diff --git a/R/ds.rNorm.R b/R/ds.rNorm.R
index 76d885f11..05bb15d20 100644
--- a/R/ds.rNorm.R
+++ b/R/ds.rNorm.R
@@ -40,8 +40,8 @@
#'
#' @param samp.size an integer value or an integer vector that defines the length
#' of the random numeric vector to be created in each source.
-#' @param mean the mean value or vector of the Normal distribution to be created.
-#' @param sd the standard deviation of the Normal distribution to be created.
+#' @param mean the mean value or vector of the Normal distribution to be created. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
+#' @param sd the standard deviation of the Normal distribution to be created. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param newobj a character string that provides the name for the output variable
#' that is stored on the data servers. Default \code{newObject}.
#' @param seed.as.integer an integer
@@ -51,15 +51,16 @@
#' If FALSE it will only return the trigger seed value you have provided.
#' Default is FALSE.
#' @param force.output.to.k.decimal.places an integer vector that
-#' forces the output random numbers vector to have k decimals.
+#' forces the output random numbers vector to have k decimals. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.rNorm} returns random number vectors with a normal distribution for each
-#' study, taking into account the values specified in each parameter of the function.
-#' The output vector is written to the server-side.
-#' If requested, it also returned to the client-side the full 626 lengths random seed vector
-#' generated in each source (see info for the argument \code{return.full.seed.as.set}).
+#' @return \code{ds.rNorm} writes a random number vector with a normal distribution
+#' to the server-side in each study and returns a list to the client-side containing
+#' \code{integer.seed.as.set.by.source} (the trigger seed set in each source),
+#' \code{random.vector.length.by.source} (the length of the vector created in each source)
+#' and, if \code{return.full.seed.as.set} is TRUE, \code{full.seed.as.set}
+#' (the full 626 length random seed vector generated in each source).
#' @examples
#' \dontrun{
#'
@@ -92,7 +93,7 @@
#'
#' ds.rNorm(samp.size=c(10,20,45), #the length of the vector created in each source is different
#' mean=c(1,6,4), #the mean of the Normal distribution changes in each server
-#' sd=as.character(c(1,4,3)), #the sd of the Normal distribution changes in each server
+#' sd=c(1,4,3), #the sd of the Normal distribution changes in each server
#' newobj="Norm.dist",
#' seed.as.integer=2345,
#' return.full.seed.as.set=FALSE,
@@ -114,20 +115,12 @@
#' datashield.logout(connections)
#' }
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.rNorm<-function(samp.size=1,mean=0,sd=1, newobj="newObject", seed.as.integer=NULL, return.full.seed.as.set=FALSE,
force.output.to.k.decimal.places=9,datasources=NULL){
-##################################################################################
-# look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
########################
#TEST SEED PRIMING VALUE
@@ -169,9 +162,14 @@ mess2<-("ERROR: appropriate values must be set for samp.size, mean, sd, and newo
return(mess2)
}
+numsources<-length(datasources)
+mean<-.expand_to_studies(mean, "mean", numsources)
+sd<-.expand_to_studies(sd, "sd", numsources)
+force.output.to.k.decimal.places<-.expand_to_studies(force.output.to.k.decimal.places, "force.output.to.k.decimal.places", numsources)
+
sd.valid<-1
if(is.numeric(sd)){
- if(sd<=0){
+ if(any(sd<=0)){
sd.valid<-0
}
}
@@ -182,7 +180,7 @@ return(mess3)
}
decimal.places.valid<-1
-if(force.output.to.k.decimal.places<0||force.output.to.k.decimal.places>9){
+if(any(force.output.to.k.decimal.places<0|force.output.to.k.decimal.places>9)){
decimal.places.valid<-0
}
@@ -220,8 +218,7 @@ if(seed.as.text=="NULL"){
message("NO SEED SET IN STUDY",study.id,"\n\n")
}
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
- ssDS.obj[[study.id]] <- DSI::datashield.aggregate(datasources[study.id], as.symbol(calltext))
+ ssDS.obj[[study.id]] <- datashield.aggregate(datasources[study.id], call("setSeedDS", seedtext=seed.as.text))
}
message("\n\n")
@@ -237,104 +234,15 @@ samp.size<-rep(samp.size,numsources)
}
for(k in 1:numsources){
+ datashield.assign(datasources[k], newobj, call("rNormDS", samp.size[k], mean=mean[k], sd=sd[k], force.output.to.k.decimal.places=force.output.to.k.decimal.places[k]))
+}
-toAssign<-paste0("rNormDS(",samp.size[k],",",mean, ",", sd, ",", force.output.to.k.decimal.places,")")
-
-
- if(is.null(toAssign)){
- stop("Please give the name of object to assign or an expression to evaluate and assign.!\n", call.=FALSE)
- }
-
- # now do the business
-
- DSI::datashield.assign(datasources[k], newobj, as.symbol(toAssign))
- }
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors && !return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
- if(no.errors && return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(full.seed.as.set=ssDS.obj, #
- integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
+if(return.full.seed.as.set){
+return(list(full.seed.as.set=ssDS.obj,
+ integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
+}
+return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
}
diff --git a/R/ds.rPois.R b/R/ds.rPois.R
index 74be7fdf7..e6ae42d46 100644
--- a/R/ds.rPois.R
+++ b/R/ds.rPois.R
@@ -29,7 +29,7 @@
#'
#' @param samp.size an integer value or an integer vector that defines the length of the
#' random numeric vector to be created in each source.
-#' @param lambda the number of events mean per interval.
+#' @param lambda the number of events mean per interval. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param newobj a character string that provides the name for the output variable
#' that is stored on the data servers. Default \code{newObject}.
#' @param seed.as.integer an integer or a NULL value which provides the random seed
@@ -41,12 +41,12 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.rPois} returns random number vectors with a Poisson distribution for each study,
-#' taking into account the values specified in each parameter of the function.
-#' The created vectors are stored in the server-side.
-#' If requested, it also returned to the client-side the full
-#' 626 lengths random seed vector generated in each source
-#' (see info for the argument \code{return.full.seed.as.set}).
+#' @return \code{ds.rPois} writes a random number vector with a Poisson distribution
+#' to the server-side in each study and returns a list to the client-side containing
+#' \code{integer.seed.as.set.by.source} (the trigger seed set in each source),
+#' \code{random.vector.length.by.source} (the length of the vector created in each source)
+#' and, if \code{return.full.seed.as.set} is TRUE, \code{full.seed.as.set}
+#' (the full 626 length random seed vector generated in each source).
#'
#' @examples
#'
@@ -81,7 +81,7 @@
#'
#' # Generating the vectors in the Opal servers
#' ds.rPois(samp.size=c(13,20,25), #the length of the vector created in each source is different
-#' lambda=as.character(c(2,3,4)), #different mean per interval (2,3,4) in each source
+#' lambda=c(2,3,4), #different mean per interval (2,3,4) in each source
#' newobj="Pois.dist",
#' seed.as.integer=1234,
#' return.full.seed.as.set=FALSE,
@@ -98,19 +98,11 @@
#' datashield.logout(connections)
#' }
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.rPois<-function(samp.size=1,lambda=1, newobj="newObject", seed.as.integer=NULL, return.full.seed.as.set=FALSE, datasources=NULL){
-##################################################################################
-# look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
########################
#TEST SEED PRIMING VALUE
@@ -152,9 +144,12 @@ mess2<-("ERROR: appropriate values must be set for samp.size, lambda, and newobj
return(mess2)
}
+numsources<-length(datasources)
+lambda<-.expand_to_studies(lambda, "lambda", numsources)
+
lambda.valid<-1
if(is.numeric(lambda)){
- if(lambda<=0){
+ if(any(lambda<=0)){
lambda.valid<-0
}
}
@@ -193,8 +188,7 @@ if(seed.as.text=="NULL"){
message("NO SEED SET IN STUDY",study.id,"\n\n")
}
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
- ssDS.obj[[study.id]] <- DSI::datashield.aggregate(datasources[study.id], as.symbol(calltext))
+ ssDS.obj[[study.id]] <- datashield.aggregate(datasources[study.id], call("setSeedDS", seedtext=seed.as.text))
}
message("\n\n")
@@ -210,104 +204,15 @@ samp.size<-rep(samp.size,numsources)
}
for(k in 1:numsources){
+ datashield.assign(datasources[k], newobj, call("rPoisDS", samp.size[k], lambda=lambda[k]))
+}
-toAssign<-paste0("rPoisDS(",samp.size[k],",",lambda, ")")
-
-
- if(is.null(toAssign)){
- stop("Please give the name of object to assign or an expression to evaluate and assign.!\n", call.=FALSE)
- }
-
- # now do the business
-
- DSI::datashield.assign(datasources[k], newobj, as.symbol(toAssign))
- }
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors && !return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
- if(no.errors && return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(full.seed.as.set=ssDS.obj, #
- integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
+if(return.full.seed.as.set){
+return(list(full.seed.as.set=ssDS.obj,
+ integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
+}
+return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
}
diff --git a/R/ds.rUnif.R b/R/ds.rUnif.R
index ea74766b5..bac2db04b 100644
--- a/R/ds.rUnif.R
+++ b/R/ds.rUnif.R
@@ -43,10 +43,10 @@
#'
#' @param samp.size an integer value or an integer vector that defines the
#' length of the random numeric vector to be created in each source.
-#' @param min a numeric scalar that specifies the minimum value of the
-#' random numbers in the distribution.
-#' @param max a numeric scalar that specifies the maximum value of the
-#' random numbers in the distribution.
+#' @param min a numeric value that specifies the minimum value of the
+#' random numbers in the distribution. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
+#' @param max a numeric value that specifies the maximum value of the
+#' random numbers in the distribution. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#' @param newobj a character string that provides the name for the output variable
#' that is stored on the data servers. Default \code{newObject}.
#' @param seed.as.integer an integer or a NULL value which provides the random
@@ -56,16 +56,17 @@
#' return the trigger seed value you have provided. Default is FALSE.
#' @param force.output.to.k.decimal.places an integer or
#' an integer vector that forces the output random
-#' numbers vector to have k decimals.
+#' numbers vector to have k decimals. A single value is used in every study; a vector must have one value per study, with its k-th value used in study k.
#'
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}} objects obtained after login.
#' If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.Unif} returns random number vectors with a uniform distribution for each study,
-#' taking into account the values specified in each parameter of the function.
-#' The created vectors are stored in the server-side. If requested, it also returned to the
-#' client-side the full 626 lengths random seed vector generated in each source
-#' (see info for the argument \code{return.full.seed.as.set}).
+#' @return \code{ds.rUnif} writes a random number vector with a uniform distribution
+#' to the server-side in each study and returns a list to the client-side containing
+#' \code{integer.seed.as.set.by.source} (the trigger seed set in each source),
+#' \code{random.vector.length.by.source} (the length of the vector created in each source)
+#' and, if \code{return.full.seed.as.set} is TRUE, \code{full.seed.as.set}
+#' (the full 626 length random seed vector generated in each source).
#' @examples
#'
#' \dontrun{
@@ -99,8 +100,8 @@
#' # Generating the vectors in the Opal servers
#'
#' ds.rUnif(samp.size = c(12,20,4), #the length of the vector created in each source is different
-#' min = as.character(c(0,2,5)), #different minumum value of the function in each source
-#' max = as.character(c(2,5,9)), #different maximum value of the function in each source
+#' min = c(0,2,5), #different minumum value of the function in each source
+#' max = c(2,5,9), #different maximum value of the function in each source
#' newobj = "Unif.dist",
#' seed.as.integer = 234,
#' return.full.seed.as.set = FALSE,
@@ -122,20 +123,12 @@
#' }
#'
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.rUnif<-function(samp.size=1,min=0,max=1, newobj="newObject", seed.as.integer=NULL, return.full.seed.as.set=FALSE,
force.output.to.k.decimal.places=9,datasources=NULL){
-##################################################################################
-# look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
########################
#TEST SEED PRIMING VALUE
@@ -178,11 +171,16 @@ mess2<-("ERROR: appropriate values must be set for samp.size, min, max, and newo
return(mess2)
}
+numsources<-length(datasources)
+min<-.expand_to_studies(min, "min", numsources)
+max<-.expand_to_studies(max, "max", numsources)
+force.output.to.k.decimal.places<-.expand_to_studies(force.output.to.k.decimal.places, "force.output.to.k.decimal.places", numsources)
+
minmax.valid<-1
if(is.numeric(min) && is.numeric(max)){
- if(min>=max){
+ if(any(min>=max)){
minmax.valid<-0
}
@@ -194,7 +192,7 @@ return(mess3)
}
decimal.places.valid<-1
-if(force.output.to.k.decimal.places<0||force.output.to.k.decimal.places>9){
+if(any(force.output.to.k.decimal.places<0|force.output.to.k.decimal.places>9)){
decimal.places.valid<-0
}
@@ -235,10 +233,9 @@ if(seed.as.text=="NULL"){
message("NO SEED SET IN STUDY",study.id,"\n")
} else {
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
- ssDS.obj[[study.id]] <- DSI::datashield.aggregate(datasources[study.id], as.symbol(calltext))
+ ssDS.obj[[study.id]] <- datashield.aggregate(datasources[study.id], call("setSeedDS", seedtext=seed.as.text))
+}
}
-}
##############################
@@ -249,104 +246,15 @@ samp.size<-rep(samp.size,numsources)
}
for(k in 1:numsources){
+ datashield.assign(datasources[k], newobj, call("rUnifDS", samp.size[k], min=min[k], max=max[k], force.output.to.k.decimal.places=force.output.to.k.decimal.places[k]))
+}
-toAssign<-paste0("rUnifDS(",samp.size[k],",",min, ",", max, ",", force.output.to.k.decimal.places,")")
-
-
- if(is.null(toAssign)){
- stop("Please give the name of object to assign or an expression to evaluate and assign.!\n", call.=FALSE)
- }
-
- # now do the business
-
- DSI::datashield.assign(datasources[k], newobj, as.symbol(toAssign))
- }
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors && !return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
- if(no.errors && return.full.seed.as.set){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(full.seed.as.set=ssDS.obj, #
- integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size, #
- is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
+if(return.full.seed.as.set){
+return(list(full.seed.as.set=ssDS.obj,
+ integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
+}
+return(list(integer.seed.as.set.by.source=single.integer.seed,random.vector.length.by.source=samp.size))
}
diff --git a/R/ds.reShape.R b/R/ds.reShape.R
index f2214f559..38fa36b5f 100644
--- a/R/ds.reShape.R
+++ b/R/ds.reShape.R
@@ -32,12 +32,10 @@
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
-#' @return \code{ds.reShape} returns to the server-side a reshaped data frame
-#' converted from 'long' to 'wide' format or from 'wide' to long' format.
-#' Also, two validity messages are returned to the client-side
-#' indicating whether the new object has been created in each data source and if so whether
-#' it is in a valid form.
+#' @return \code{ds.reShape} returns to the server-side a reshaped data frame
+#' converted from 'long' to 'wide' format or from 'wide' to long' format.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#'
@@ -84,15 +82,7 @@
ds.reShape <- function(data.name=NULL, varying=NULL, v.names=NULL, timevar.name="time", idvar.name="id",
drop=NULL, direction=NULL, sep=".", newobj="newObject", datasources=NULL){
- # look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
if(is.null(data.name)){
stop("Please provide the name of the list that holds the input vectors!", call.=FALSE)
@@ -125,81 +115,5 @@ ds.reShape <- function(data.name=NULL, varying=NULL, v.names=NULL, timevar.name=
calltext <- call("reShapeDS", data.name, varying.transmit, v.names.transmit, timevar.name, idvar.name, drop.transmit, direction, sep)
DSI::datashield.assign(datasources, newobj, calltext)
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- # #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.reShape
diff --git a/R/ds.sample.R b/R/ds.sample.R
index 9bd6780fb..e52af826c 100644
--- a/R/ds.sample.R
+++ b/R/ds.sample.R
@@ -123,27 +123,14 @@
#' progress. The default value for notify.of.progress is FALSE.
#' @return the object specified by the argument (or default name
#' 'newobj.sample')
-#' which is written to the serverside. In addition, two validity messages are returned
-#' indicating whether has been created in each data source and if so whether
-#' it is in a valid form. If its form is not valid in at least one study - e.g. because
-#' a disclosure trap was tripped and creation of the full output object was blocked -
-#' ds.dataFrameSort() also returns any studysideMessages that may explain the error in creating
-#' the full output object. We are currently working to extend the information that can
-#' be returned to the clientside when an error occurs.
+#' which is written to the serverside.
#' @author Paul Burton, for DataSHIELD Development Team, 15/4/2020
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
ds.sample<-function(x=NULL, size=NULL, seed.as.integer=NULL, replace=FALSE, prob = NULL, newobj=NULL,datasources=NULL, notify.of.progress=FALSE){
- # if no opal login details are provided look for 'opal' objects in the environment
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
# check if a value has been provided for x
if(is.null(x)){
@@ -210,12 +197,10 @@ if(seed.as.text=="NULL"){
message("NO SEED SET IN STUDY",study.id,"\n\n")
}
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
-
if (notify.of.progress)
- message(calltext)
-
- ssDS.obj[[study.id]] <- DSI::datashield.aggregate(datasources[study.id], as.symbol(calltext))
+ message("setSeedDS(", seed.as.text, ")")
+
+ ssDS.obj[[study.id]] <- datashield.aggregate(datasources[study.id], call("setSeedDS", seedtext=seed.as.text))
}
if (notify.of.progress)
message("\n\n")
@@ -232,87 +217,6 @@ calltext <- call("sampleDS", x.transmit=x, size.transmit=size, replace.transmit=
DSI::datashield.assign(datasources, newobj, calltext)
-
-#############################################################################################################
-#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
- #
-#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
-test.obj.name<-newobj #
- #
-#TRACER #
-#return(test.obj.name) #
-#} #
- #
- #
-# CALL SEVERSIDE FUNCTION #
-calltext <- call("testObjExistsDS", test.obj.name) #
- #
-object.info<-DSI::datashield.aggregate(datasources, calltext) #
- #
-# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
-# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
-num.datasources<-length(object.info) #
- #
- #
-obj.name.exists.in.all.sources<-TRUE #
-obj.non.null.in.all.sources<-TRUE #
- #
-for(j in 1:num.datasources){ #
- if(!object.info[[j]]$test.obj.exists){ #
- obj.name.exists.in.all.sources<-FALSE #
- } #
- if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
- obj.non.null.in.all.sources<-FALSE #
- } #
- } #
- #
-if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
- #
- return.message<- #
- paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
- #
- #
- }else{ #
- #
- return.message.1<- #
- paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
- #
- return.message.2<- #
- paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
- #
- return.message.3<- #
- paste0("Please use ds.ls() to identify where missing") #
- #
- #
- return.message<-list(return.message.1,return.message.2,return.message.3) #
- #
- } #
- #
- calltext <- call("messageDS", test.obj.name) #
- studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
- #
- no.errors<-TRUE #
- for(nd in 1:num.datasources){ #
- if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
- no.errors<-FALSE #
- } #
- } #
- #
- #
- if(no.errors){ #
- validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
- return(list(is.object.created=return.message,validity.check=validity.check)) #
- } #
- #
-if(!no.errors){ #
- validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
- return(list(is.object.created=return.message,validity.check=validity.check, #
- studyside.messages=studyside.message)) #
- } #
- #
-#END OF CHECK OBJECT CREATED CORECTLY MODULE #
-#############################################################################################################
-
}
#ds.sample
diff --git a/R/ds.setSeed.R b/R/ds.setSeed.R
index ee84a1740..acd73f810 100644
--- a/R/ds.setSeed.R
+++ b/R/ds.setSeed.R
@@ -41,6 +41,7 @@
#' each source and also the integer vector of 626 elements
#' that is \code{.Random.seed itself}.
#' @author DataSHIELD Development Team
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @examples
#' \dontrun{
#' ## Version 6, for version 5 see the Wiki
@@ -86,16 +87,7 @@
#' @export
ds.setSeed<-function(seed.as.integer=NULL,datasources=NULL){
-##################################################################################
-# look for DS connections
- if(is.null(datasources)){
- datasources <- datashield.connections_find()
- }
-
- # ensure datasources is a list of DSConnection-class
- if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
- stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
- }
+ datasources <- .set_datasources(datasources)
seed.valid<-0
@@ -116,8 +108,7 @@ mess1<-("ERROR terminated: seed.as.integer must be set as an integer [numeric] o
return(mess1)
}
- calltext <- paste0("setSeedDS(", seed.as.text, ")")
- ssDS.obj <- DSI::datashield.aggregate(datasources, as.symbol(calltext))
+ ssDS.obj <- datashield.aggregate(datasources, call("setSeedDS", seedtext=seed.as.text))
return.message<-paste0("Trigger integer to prime random seed = ",seed.as.text)
diff --git a/R/utils.R b/R/utils.R
index 6c5a23645..019f55a3c 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -113,3 +113,25 @@
cli_abort("Please provide the name of a data.frame or matrix!", call.=FALSE)
}
}
+
+#' Expand an Argument to One Value per Study
+#'
+#' Arguments that may differ between studies can be given either as a single
+#' value, used for every study, or as a vector with one value per study.
+#'
+#' @param values A vector of length 1 or `num_studies`.
+#' @param arg_name The name of the argument, used in the error message.
+#' @param num_studies The number of studies being analysed.
+#' @importFrom cli cli_abort
+#' @return A vector of length `num_studies`.
+#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
+#' @noRd
+.expand_to_studies <- function(values, arg_name, num_studies) {
+ if (length(values) == 1) {
+ return(rep(values, num_studies))
+ }
+ if (length(values) != num_studies) {
+ cli_abort("'{arg_name}' must be length 1 or one value per study")
+ }
+ return(values)
+}
diff --git a/README.md b/README.md
index 616c30dcd..721593bd1 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ You can install the released version of dsBaseClient from
[CRAN](https://cran.r-project.org/package=dsBaseClient) with:
``` r
-install.packages("dsBaseClient")
+install.packages("dsBaseClient")
```
And the development version from
@@ -23,8 +23,8 @@ And the development version from
install.packages("remotes")
remotes::install_github("datashield/dsBaseClient", "")
-# Install v6.3.5 with the following
-remotes::install_github("datashield/dsBaseClient", "6.3.5")
+# Install v7.0.0 with the following
+remotes::install_github("datashield/dsBaseClient", "7.0.0")
```
For a full list of development branches, checkout https://github.com/datashield/dsBaseClient/branches
diff --git a/REFACTOR_GUIDE.md b/REFACTOR_GUIDE.md
deleted file mode 100644
index f7226e83e..000000000
--- a/REFACTOR_GUIDE.md
+++ /dev/null
@@ -1,449 +0,0 @@
-# Refactoring Plan: dsBase & dsBaseClient Function Pairs
-
-> **Action:** Replace `/Users/tcadman/github-repos/ds-core/dsBaseClient/REFACTOR_GUIDE.md` with this plan content so it's accessible across branches.
-
-## Context
-
-The `ds.colnames` / `colnamesDS` pair has been refactored as a reference implementation. The pattern shifts server-state validation (object existence, type checking) from client to server, reducing network round trips and centralizing validation where data lives. This needs to be applied across all remaining function pairs in both packages.
-
-The refactored `ds.colnames` branch (`v7.0-dev-colnames`) also introduces shared helpers:
-- **Client:** `R/utils.R` with `.set_datasources()`, `.check_df_name_provided()`
-- **Server:** `R/utils.R` with `.loadServersideObject()`, `.checkClass()`
-
-## Relationship Between Packages
-
-- **dsBaseClient** (`/Users/tcadman/github-repos/ds-core/dsBaseClient/R/`) — Client functions (`ds.functionName`) that validate inputs and dispatch calls to server
-- **dsBase** (`/Users/tcadman/github-repos/ds-core/dsBase/R/`) — Server functions (`functionNameDS`) that execute on the data
-
-## What Changes Per Function Pair
-
-### Client-side (dsBaseClient)
-
-1. **Replace datasource boilerplate** with `datasources <- .set_datasources(datasources)`
- - Removes: `datashield.connections_find()` + DSConnection class check (~8 lines)
-
-2. **Remove `isDefined()` calls** — server handles via `.loadServersideObject()`
-
-3. **Remove `checkClass()` calls and subsequent type guards** — server handles via `.checkClass()`
-
-4. **Add `classConsistencyCheck` parameter** — For any function where the input accepts more than one permitted class, add a `classConsistencyCheck` parameter. The server function returns `class = class(obj)` in its result list; the client checks consistency via `.checkClassConsistency()` when the parameter is TRUE, then strips the `class` field before returning to the user. Rules for the default value:
- - **TRUE** when permitted classes include genuinely different types (e.g. data.frame + matrix, factor + character + integer)
- - **FALSE** when permitted classes are only `numeric` and `integer` (these are effectively interchangeable)
- - **No parameter** when only one class is permitted (e.g. `ds.levels` only permits factor — consistency is guaranteed by `.checkClass()`)
-
- **Verification:** after refactoring, inspect every `return()` in the client function — `class` (or `class.x` / `class.index` for multi-input functions) must not appear as a named element of the returned list. Stripping can be explicit (`r$class <- NULL`) or implicit (building the return from a specific subset of fields), but the absence of `class` in the final returned value must be visible at the `return()` site.
-
-5. **Remove `ValidityMessage`** — Server functions that returned `ValidityMessage = "VALID ANALYSIS"` should remove it. Failures should call `stop()` instead of returning a failure message. Remove `ValidityMessage` from client returns too. This is a major release so API changes are acceptable.
-
-6. **Remove `isAssigned()` calls** — no longer verify object creation client-side
-
-7. **Remove MODULE 5 boilerplate** — the ~40-80 line "CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED" block
-
-8. **Remove `checks` parameter** — functions like `ds.dim` and `ds.length` have a `checks` parameter that gates `isDefined()`/`checkClass()` calls. Once those calls are removed, the parameter serves no purpose. Remove it from the function signature and delete the associated conditional block.
-
-9. **Replace per-study loops with single aggregate calls** — some functions (e.g. `ds.isNA`) loop over datasources one at a time (`datashield.aggregate(datasources[i], ...)`). Since `datashield.aggregate` already supports multiple datasources and returns a named list, replace these loops with a single call and process results client-side. This collapses N sequential round trips into 1 parallel call.
-
-10. **Keep**: null-input checks (or replace with `.check_df_name_provided()`), default `newobj` naming, the actual server call dispatch, any pure client-side logic
-
-### Server-side (dsBase)
-
-Two refactor patterns, depending on how the server function currently receives its input. Pick the one that matches.
-
-**Pattern A — function already receives a string name and uses `eval(parse())` internally.**
-
-1. Replace `eval(parse(text=x), envir=parent.frame())` with `.loadServersideObject(x)`.
-2. Add `.checkClass(obj = x.val, obj_name = x, permitted_classes = …)` right after loading, where the client previously enforced type constraints.
-3. Keep all computation, disclosure controls, privacy checks untouched.
-
-**Pattern B — function currently receives a resolved R object via dispatch-layer evaluation (`as.symbol()` or `call()` in the client).**
-
-1. Rename the function parameter from its descriptive body-variable name (e.g. `xvect`, `X`) to a simple string-name parameter (e.g. `x`). Do **not** rename the body-variable usages.
-2. At the top of the body, load into the original body-variable name: `xvect <- .loadServersideObject(x)`.
-3. Add `.checkClass(obj = xvect, obj_name = x, permitted_classes = …)`.
-4. On the client, switch dispatch from `as.symbol(paste0("funcDS(", x, ")"))` to `call("funcDS", x)` so the string is passed through instead of being evaluated.
-5. Update the `@param` roxygen line to describe the string-name form.
-
-Both patterns leave the function body untouched — **no renaming inside the body, no restyling.** Minimise diff.
-
-### Returning class from the server
-
-Some client functions previously called `checkClass()` purely to drive **client-side routing** (e.g. decide which server function to dispatch, or which output format to use, or to warn about an argument being ignored). That's a separate network round trip solely to discover the class of the input object — redundant, because the server that runs the aggregate already has the object in hand.
-
-The batch 2 precedent (`dimDS`, `lengthDS`) is: **return the class as a field of the aggregate result**. For example:
-
-```r
-lengthDS <- function(x){
- x.val <- .loadServersideObject(x)
- .checkClass(obj = x.val, obj_name = x, permitted_classes = c(...))
- list(length = length(x.val), class = class(x.val))
-}
-```
-
-The client then reads `result$class` for any post-hoc routing, consistency check, or warning — no extra `checkClass()` call needed.
-
-Cross-study class consistency is checked via the shared helper `.checkClassConsistency(results)` (in `dsBaseClient/R/utils.R`), which aborts if the `class` field differs across studies. Always use this helper instead of inlining the check.
-
-**When to apply this:**
-- The client was previously calling `checkClass()` to select output format, format warnings, or check class consistency across studies.
-- The class information can be derived from the aggregate's input.
-
-**When not to apply this:**
-- The client needs the class *before* choosing which server function to call (true composite dispatchers like `ds.summary`, which branch to completely different server functions per class). These still need a pre-call class lookup, for example via `call("classDS", x)` as a single lightweight aggregate.
-- The class is genuinely irrelevant to the client after the call.
-
-Prefer this pattern over keeping a client-side `checkClass()` call whenever a client function currently does both a `checkClass()` and an aggregate call on the same object.
-
-### Minimal-diff rule
-
-Refactors should change as little as possible. Do not rename variables, restyle comments, reformat whitespace, or bundle unrelated cleanups. If a variable rename is stylistically tempting but not strictly required by the refactor, skip it. If the user pushes back on a change, stop and ask — do not iterate with more edits on the same file.
-
-### Do not change existing behaviour
-
-The refactor must not alter which input types a function accepts or what it returns (beyond adding `class` to the return list). If the original function accepted data.frames, the refactored version must too. Check previous behaviour before setting permitted classes in `.checkClass()`.
-
-This extends to **test coverage**. If a refactor removes output fields that tests asserted on, the refactor is not done until equivalent coverage is added. Do not merely delete assertions to make tests pass — that silently reduces what the suite verifies. The change is reviewable by diffing `test-smk-*.R` against the base branch: every removed `expect_*` must be either (a) redundant because the same behaviour is covered elsewhere in the same file, or (b) replaced with an assertion covering the same server-side behaviour.
-
-### Adding new parameters to existing exported functions
-
-When adding a parameter to an already-released function (e.g. `classConsistencyCheck`, a new behaviour flag), **place it after all existing named parameters** — never in the middle of the signature. Inserting a parameter mid-signature silently breaks every caller that used positional argument order for anything to its right. Append it to the end (after `datasources=NULL` is acceptable even though `datasources` is conventionally last), and document the default value in `@param`.
-
-### Tests
-
-**Server-side unit tests** (new `test-smk-functionNameDS.R` in dsBase):
-- Happy path: call with valid input, assert correct output
-- Unhappy: nonexistent object → `expect_error(..., "does not exist")`
-- Unhappy: wrong type → `expect_error(..., "must be of type")` (only where `.checkClass()` is used)
-
-**Client-side end-to-end tests** (update existing `test-smk-ds.functionName.R` in dsBaseClient):
-- Happy path: existing tests should still pass
-- Unhappy: nonexistent object → `expect_error(..., "DataSHIELD errors")`
-- Unhappy: wrong type → `expect_error(..., "DataSHIELD errors")` (where type was previously checked client-side)
-- Update any tests that expected client-side error messages to expect server-originated errors
-- **When MODULE 5 assertions are removed, add comparable replacements.** The old MODULE 5 block returned `$is.object.created` and `$validity.check` messages asserting that `newobj` existed on every server. When those assertions are stripped, add equivalent checks inside the same `test_that` block that verify the object was created on all sources — e.g. `ds_expect_variables(c(""))` or `expect_no_error(ds.class(""))`. Relying on the shutdown-block `ds_expect_variables()` alone is not sufficient because it can't pinpoint which test created the missing object.
-
-**Client-side smoke tests** (new `test-smk-ds.functionName.R` if none exists):
-- If no smoke test file exists for a refactored client function, create one. Every refactored function must have at least a basic happy-path smoke test that exercises the server call and verifies the result.
-- Follow the existing test pattern: `connect.studies.dataset.cnsim(...)`, `test_that("setup", ...)`, main test block, `test_that("shutdown", ...)`, `disconnect.studies.dataset.cnsim()`.
-
-**Client-side performance tests** (new `test-perf-ds.functionName.R` in dsBaseClient):
-- Add a performance test for each refactored client function. Follow the pattern in `test-perf-ds.class.R`: call the function in a timed loop, compare against a reference rate from the perf profile CSV.
-- Run with `PERF_DURATION_SEC=2 devtools::test(filter = "perf-")` during development; the default 30-second duration is for CI.
-- **Do not** include Arjuna Technologies copyright headers in new test files. The existing headers in pre-refactor files should be left as-is, but new files we create should not carry third-party copyright.
-- **The perf test must replicate the smoke test.** Before writing a perf test, read the corresponding `test-smk-ds.functionName.R` and copy:
- 1. The `connect.studies.dataset.*()` line (same dataset, same columns)
- 2. The `disconnect.studies.dataset.*()` line
- 3. The function call (same parameters, same column names, same argument names)
-
- The perf test should exercise the same code path as the smoke test's happy-path call. Do not use generic placeholder calls or different datasets.
-
-**Design decisions:**
-- Functions accepting any class: use `.loadServersideObject()` only, no `.checkClass()`
-- Client tests must include unhappy paths testing server error propagation
-- Start with Batch 1 (simple coercions)
-
-### Authorship
-
-After the refactor commits for a batch have landed, add `Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands` as a new `@author` roxygen line in every R/ file touched on each branch (dsBase and dsBaseClient), matching the existing `@author` line(s) below:
-
-```
-#' @author
-#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
-```
-
-Skip files that have no existing `@author` line (e.g. `R/utils.R`). Do this as a separate trailing commit per repo with message `docs: updated authorship`, not bundled with the refactor commits.
-
-**Only add the tag to files you actually refactored** (replaced `eval(parse())`, added `.loadServersideObject` / `.checkClass`, replaced MODULE 5, converted dispatch to `call()`, etc.). If a file in a batch's function list turns out not to need any substantive change — for example a server function whose inputs are client-transmitted literal data rather than object names (`dmtC2SDS` is one such case) — leave its author line as-is. Adding `@author Tim Cadman` to an untouched file is incorrect authorship attribution.
-
-## Excluded Functions
-
-**Deprecated (12):** ds.look, ds.meanByClass, ds.message, ds.recodeLevels, ds.setDefaultOpals, ds.subset, ds.subsetByClass, ds.table1D, ds.table2D, ds.vectorCalc, ds.listOpals, ds.listServersideFunctions
-
-**Already done:** ds.colnames / colnamesDS
-
-**Client-only (no server pair):** checkClass.R, isDefined.R, isAssigned.R, extract.R, glmChecks.R, getPooledMean.R, getPooledVar.R, helpers (meanByClassHelper*, subsetHelper, logical2int, colPercent, rowPercent)
-
-## Batches
-
-### Batch 1 — Simple Type Coercions (11 pairs)
-Single input → single output, straightforward `eval(parse())` replacement.
-
-**TODO:** Add `classConsistencyCheck` parameter to batch-1 functions with >1 permitted class. For numeric/integer-only functions (ds.abs, ds.exp, ds.log, ds.sqrt) default to FALSE; for others (ds.asDataMatrix: data.frame/matrix, ds.asLogical: numeric/integer/character/matrix) default to TRUE. Server functions need to return class in results to support this. dsBase batch-1 is already merged — server changes need a new branch or inclusion in a later batch.
-
-**TODO:** `test-smk-asLogicalDS.R` in dsBase is missing a wrong-type test case (e.g. passing a list). Other `.checkClass()` functions (absDS, expDS, logDS, sqrtDS) have this test. dsBase batch-1 is merged — fix in a follow-up.
-
-| Client | Server | Permitted classes | Notes |
-|--------|--------|-------------------|-------|
-| ds.abs | absDS | numeric, integer | |
-| ds.asCharacter | asCharacterDS | * | |
-| ds.asDataMatrix | asDataMatrixDS | data.frame, matrix | |
-| ds.asInteger | asIntegerDS | * | |
-| ds.asList | asListDS | * | **AGGREGATE** (not assign); server takes 2 params (x.name, newobj) |
-| ds.asLogical | asLogicalDS | * | Server has existing type validation (numeric/integer/character/matrix) — preserve as `.checkClass()` |
-| ds.asMatrix | asMatrixDS | * | |
-| ds.asNumeric | asNumericDS | * | Server has complex factor/character conversion logic — preserve |
-| ds.exp | **NEW: expDS** | numeric, integer | No server DS function exists — client currently calls native `exp()` via `as.symbol()`. Must create `expDS.R` |
-| ds.log | **NEW: logDS** | numeric, integer | No server DS function exists — client currently calls native `log()` via `as.symbol()`. Must create `logDS.R`. Has `base` parameter |
-| ds.sqrt | sqrtDS | numeric, integer | |
-
-`*` = accept any class — only use `.loadServersideObject()`, no `.checkClass()` needed
-
-**Batch 1 sub-patterns discovered:**
-- **Math ops (abs, exp, log, sqrt):** Client uses `checkClass()` + `isAssigned()`, no MODULE 5
-- **Type conversions (asCharacter, asDataMatrix, asInteger, asLogical, asMatrix, asNumeric):** Client uses `isDefined()` + MODULE 5 block (except asList which has neither)
-- **asList is unique:** Uses `datashield.aggregate` instead of `datashield.assign`
-
-### Batch 2 — Simple Aggregations (10 pairs)
-Return results to client, no server-side assignment.
-
-| Client | Server | Permitted classes |
-|--------|--------|-------------------|
-| ds.class | classDS | * |
-| ds.dim | dimDS | data.frame, matrix |
-| ds.length | lengthDS | character, factor, integer, logical, numeric, list |
-| ds.names | namesDS | * |
-| ds.isNA | isNaDS | character, factor, integer, logical, numeric, data.frame, matrix |
-| ds.numNA | numNaDS | * |
-| ds.ls | lsDS | (no object input) |
-| ds.completeCases | completeCasesDS | * (no .checkClass — server handles via own branching) |
-| ds.levels | levelsDS | factor |
-| ds.unique | uniqueDS | * |
-
-**Deferred from Batch 2:** ds.isValid / isValidDS — `isValidDS` is used as an internal disclosure-control helper by `replaceNaDS` (Batch 4), `quantileMeanDS` (Batch 3), and `rowColCalcDS` (Batch 10), all passing objects directly. Cannot change `isValidDS` signature until those callers are refactored. Refactor ds.isValid/isValidDS when the last internal caller is refactored (see Batch 10 notes).
-
-**Batch 2 sub-patterns:**
-- **Standard eval(parse()) functions (classDS, dimDS, lengthDS, namesDS, lsDS, completeCasesDS, uniqueDS):** Server uses `eval(parse(text=x), envir=parent.frame())` — replace with `.loadServersideObject()`
-- **Dispatch-layer resolution functions (isNaDS, numNaDS, levelsDS):** Server receives resolved R objects via client `as.symbol()`/`call()` dispatch — change server to accept string name + `.loadServersideObject()`, change client to `call("funcDS", x)`
-- **Assign functions (completeCases, unique):** Use `datashield.assign` not `datashield.aggregate` — still remove MODULE 5 / isAssigned
-- **Client-side processing to preserve:** ds.dim and ds.length have `type` parameter with alias normalization and cross-study pooling; ds.isNA has per-study loop with conditional messaging; ds.ls has wildcard `*` → `_:A:_` escaping
-- **Pooling functions (dimDS, lengthDS):** Return `list(dim=..., class=...)` / `list(length=..., class=...)` so client can check cross-study class consistency before pooling results
-
-### Batch 3 — Statistics (10 pairs)
-Aggregate functions returning computed values. Some have multi-step server calls.
-
-| Client | Server | Notes |
-|--------|--------|-------|
-| ds.mean | meanDS | has disclosure controls |
-| ds.var | varDS | has disclosure controls |
-| ds.cor | corDS | two inputs |
-| ds.corTest | corTestDS | two inputs |
-| ds.cov | covDS | two inputs |
-| ds.kurtosis | kurtosisDS1/DS2 | multi-step |
-| ds.skewness | skewnessDS1/DS2 | multi-step |
-| ds.quantileMean | quantileMeanDS | aggregate |
-| ds.meanSdGp | meanSdGpDS | aggregate |
-| ds.summary | (check server) | aggregate |
-
-### Batch 4 — Data Manipulation / Assign (15 pairs)
-Create/modify server objects. Many have MODULE 5 blocks.
-
-| Client | Server | Notes |
-|--------|--------|-------|
-| ds.Boole | BooleDS | assign, MODULE 5 |
-| ds.c | cDS | multi-input assign |
-| ds.cbind | cbindDS | multi-input, permissive check |
-| ds.rbind | rbindDS | multi-input |
-| ds.dataFrame | dataFrameDS | multi-input, complex |
-| ds.dataFrameSort | dataFrameSortDS | assign, MODULE 5 |
-| ds.dataFrameSubset | dataFrameSubsetDS1/DS2 | multi-step |
-| ds.dataFrameFill | dataFrameFillDS | assign |
-| ds.list | listDS | assign |
-| ds.unList | unListDS | assign |
-| ds.merge | mergeDS | assign, MODULE 5 |
-| ds.rep | repDS | assign |
-| ds.seq | seqDS | assign |
-| ds.replaceNA | replaceNaDS | assign, per-source loop |
-| ds.recodeValues | recodeValuesDS | assign |
-
-### Batch 5 — Matrix Operations (8 pairs)
-
-| Client | Server |
-|--------|--------|
-| ds.matrix | matrixDS |
-| ds.matrixDet | matrixDetDS1/DS2 |
-| ds.matrixDet.report | matrixDetDS2 |
-| ds.matrixDiag | matrixDiagDS |
-| ds.matrixDimnames | matrixDimnamesDS |
-| ds.matrixInvert | matrixInvertDS |
-| ds.matrixMult | matrixMultDS |
-| ds.matrixTranspose | matrixTransposeDS |
-
-### Batch 6 — Factor & Recoding (5 pairs)
-
-| Client | Server |
-|--------|--------|
-| ds.asFactor | asFactorDS1/DS2 |
-| ds.asFactorSimple | asFactorSimpleDS |
-| ds.changeRefGroup | changeRefGroupDS |
-| ds.reShape | reShapeDS |
-| ds.dmtC2S | dmtC2SDS |
-
-### Batch 7 — Modelling (8 pairs)
-Most complex. Multiple server calls, complex validation logic.
-
-| Client | Server |
-|--------|--------|
-| ds.glm | glmDS1/DS2 |
-| ds.glmSLMA | glmSLMADS1/DS2/assign |
-| ds.glmPredict | glmPredictDS.ag/as |
-| ds.glmSummary | glmSummaryDS.ag/as |
-| ds.glmerSLMA | glmerSLMADS2/assign |
-| ds.lmerSLMA | lmerSLMADS2/assign |
-| ds.gamlss | gamlssDS |
-| ds.mice | miceDS |
-
-### Batch 8 — Random Generation & Sampling (6 pairs)
-
-| Client | Server |
-|--------|--------|
-| ds.rBinom | rBinomDS |
-| ds.rNorm | rNormDS |
-| ds.rPois | rPoisDS |
-| ds.rUnif | rUnifDS |
-| ds.sample | sampleDS |
-| ds.setSeed | setSeedDS |
-
-### Batch 9 — Plotting & Visualization (7 pairs)
-
-| Client | Server |
-|--------|--------|
-| ds.histogram | histogramDS1/DS2 |
-| ds.heatmapPlot | heatmapPlotDS |
-| ds.contourPlot | (check server name) |
-| ds.densityGrid | densityGridDS |
-| ds.scatterPlot | scatterPlotDS |
-| ds.boxPlot | (check server) |
-| ds.boxPlotGG | boxPlotGGDS |
-
-**Batch 9 note:** `ds.heatmapPlot`, `ds.contourPlot`, and `ds.densityGrid` call `rangeDS` which has **not** been refactored. These calls still use `as.symbol(paste0("rangeDS(", x, ")"))`. Once `rangeDS` is refactored (batch 10 or later), go back and update these three client functions to use `call("rangeDS", x=x)`.
-
-### Batch 10 — Splines, Tables, Misc (14 pairs)
-
-| Client | Server |
-|--------|--------|
-| ds.elspline | elsplineDS |
-| ds.lspline | lsplineDS |
-| ds.ns | nsDS |
-| ds.qlspline | qlsplineDS |
-| ds.table | tableDS/tableDS.assign/tableDS2 |
-| ds.tapply | tapplyDS |
-| ds.tapply.assign | tapplyDS.assign |
-| ds.rowColCalc | rowColCalcDS |
-| ds.make | (check server) |
-| ds.assign | (check server) |
-| ds.metadata | metadataDS |
-| ds.getWGSR | getWGSRDS |
-| ds.lexis | lexisDS1/DS2/DS3 |
-| ds.hetcor | hetcorDS |
-
-**Batch 10 dependency:** `rowColCalcDS` calls `isValidDS(result)` internally as a disclosure check. When refactoring `rowColCalcDS`, replace this with direct disclosure logic or `.loadServersideObject()` + `.checkClass()`. Once done, also refactor `ds.isValid` / `isValidDS` (deferred from Batch 2). Similarly, `replaceNaDS` (Batch 4) and `quantileMeanDS` (Batch 3) call `isValidDS()` internally — refactor those callers first before changing `isValidDS`'s signature.
-
-## Known Issues
-
-**Batch 4:** `ds.dataFrameFill` perf test cannot run — function requires columns to differ across studies, which is hard to set up in a perf loop.
-
-**Batch 6:** `ds.asFactor` and `ds.changeRefGroup` perf tests fail with server-side errors. The `asFactorDS1` aggregate call errors out. `ds.changeRefGroup` may have a known pre-existing issue. Both need investigation of the batch-6 server refactoring.
-
-**Batch 7:** `ds.gamlss` perf test fails with server-side error. May be a batch-7 refactoring issue in `gamlssDS` or a dataset availability issue (gamlss dataset may not be configured on all Armadillo instances).
-
-**Batch 8:** `ds.sample` smoke test fails at the `ds.length("newobj.sample")` call — this is because the batch-2 client PR has not been merged to v7.0-dev yet, so the old `ds.length` client code cannot handle the new `list(length=..., class=...)` return from the refactored `lengthDS`.
-
-**Batch 9:** `rangeDS` has not been refactored, so `ds.heatmapPlot`, `ds.contourPlot`, and `ds.densityGrid` still use `as.symbol(paste0("rangeDS(", x, ")"))` for `rangeDS` calls. Once `rangeDS` is refactored, update these to use `call("rangeDS", x=x)`.
-
-## Per-Batch Workflow
-
-**Important:** dsBase and dsBaseClient are separate git repos. Changes must be committed and tested in the correct order since the client depends on the server package being installed.
-
-### Step 0 — Branch bootstrap
-
-When creating a new batch branch (in either repo) from `origin/v7.0-dev`:
-
-1. **dsBaseClient:** copy `R/utils.R` from the most recently refactored client branch (e.g. `origin/refactor/perf-batch-4`). `origin/v7.0-dev` on the client does not yet contain it — it only enters `v7.0-dev` once the batch-1 or batch-2 client PR merges.
-2. **dsBaseClient:** copy `REFACTOR_GUIDE.md` from the same branch, and add `^REFACTOR_GUIDE\.md$` to `.Rbuildignore` if not already there. This keeps the guide alongside the code being refactored so rules added in later batches are visible to everyone.
-3. **dsBase:** no bootstrap copy needed — `R/utils.R` with `.loadServersideObject` / `.checkClass` is already in `origin/v7.0-dev` (merged with batch-1).
-
-Commit the bootstrap separately (message: `chore: bootstrap batch-N from batch-M`) before starting the refactor work.
-
-### Step 1 — Server-side (dsBase repo)
-1. Create feature branch from `v7.0-dev` in dsBase
-2. Refactor server functions:
- - Replace `eval(parse())` → `.loadServersideObject()`
- - Add `.checkClass()` where the client had type guards
-3. Write server-side unit tests (`test-smk-functionNameDS.R`) with happy + unhappy paths
-4. Run `devtools::check(args = '--no-tests')` and `devtools::test()` in dsBase
-5. Build package: `devtools::build()`
-
-### Step 2 — Install refactored dsBase on Armadillo
-6. Ensure `inst/DATASHIELD` has `default.datashield.privacyControlLevel="permissive"` before building. **Must be the literal string `"permissive"`** — other values like `"banana"` will not work for all functions (e.g. `levelsDS` checks for `'permissive'` explicitly).
-7. Build package: `devtools::build()` in dsBase
-8. Copy the built tar to dsBaseClient as `dsBase_7.0.0-permissive.tar.gz` (this is the filename the CI pipeline references in `armadillo_azure-pipelines.yml`)
-9. Install on local Armadillo: `armadillo.login("http://localhost:8080")` then `armadillo.install_packages(paths = "", profile = "default")`
-
-### Step 3 — Client-side (dsBaseClient repo)
-7. Create feature branch from `v7.0-dev` in dsBaseClient
-8. Ensure `R/utils.R` exists (copy from `v7.0-dev-colnames` branch if needed)
-9. Refactor client functions:
- - Replace datasource boilerplate → `.set_datasources()`
- - Remove `isDefined()`, `checkClass()`, `isAssigned()` calls
- - Remove MODULE 5 blocks
- - Replace null-input checks with `.check_df_name_provided()` where applicable
-10. Update/add client end-to-end tests with happy + unhappy paths
-11. Run `devtools::check(args = '--no-tests')` in dsBaseClient
-12. Run tests against Armadillo, **not DSLite**. Set the driver to `"ArmadilloDriver"` in `tests/testthat/connection_to_datasets/login_details.R` (default is `"DSLiteDriver"`). DSLite uses whatever dsBase is installed locally in R, which may not match the refactored version on Armadillo. Run `devtools::test(filter = "smk-|disc|arg")` for affected functions (requires refactored dsBase to be installed on Armadillo)
-
-### Step 4 — Verify
-13. Run full test suite to check no regressions
-14. Run perf tests at 30 seconds (default): `devtools::test(filter = "perf-")`
-15. Compare perf results against the v7.0-dev branch baseline to detect any regressions from the refactoring
-
-### Step 5 — Pre-merge audit (mandatory)
-
-Before marking a batch complete:
-
-1. **Diff every touched `test-smk-*.R` / `test-arg-*.R` / `test-disc-*.R` against the branch base.** For each removed `expect_*` assertion, confirm it falls into one of:
- - (a) redundant — the same behaviour is covered by another assertion still present in the same file;
- - (b) replaced — a new assertion was added that covers the same server-side behaviour (e.g. `ds_expect_variables` replacing `$is.object.created`, or `ds.summary`/`ds.class` on the newobj).
-
- Any removed assertion that doesn't fall into (a) or (b) is a coverage loss that must be restored before merge.
-
-2. **Inspect every `test_that(…)` block touched by the refactor.** Each block must still contain at least one `expect_*` assertion after the refactor. Blocks stripped to just the function call are not acceptable — add `ds_expect_variables`, `expect_no_error`, or a downstream property check.
-
-3. **Diff every signature of every exported function touched.** Confirm no parameter was added in the middle of the signature; new parameters must be at the end (see "Adding new parameters to existing exported functions" above).
-
-4. **Confirm docs match signature.** `@param` blocks present for every parameter, no stale `@param` for removed arguments, `@return` not promising MODULE 5 output fields.
-
-5. **Grep for residual patterns that should have been removed:** `isDefined(`, `isAssigned(`, `CLIENTSIDE MODULE`, `testObjExistsDS`, `is.object.created`, `validity.check`, `studyside.messages` in source files (acceptable in test files only if the MODULE 5 pattern is being deliberately preserved with a replacement).
-
-## Key Files
-
-### Reference implementation
-- Client refactored: `git show v7.0-dev-colnames:R/ds.colnames.R`
-- Server refactored: `/Users/tcadman/github-repos/ds-core/dsBase/R/colnamesDS.R`
-- Client utils: `git show v7.0-dev-colnames:R/utils.R`
-- Server utils: `/Users/tcadman/github-repos/ds-core/dsBase/R/utils.R`
-- Server tests: `/Users/tcadman/github-repos/ds-core/dsBase/tests/testthat/test-smk-colnamesDS.R`
-- Client tests: `/Users/tcadman/github-repos/ds-core/dsBaseClient/tests/testthat/test-smk-ds.colnames.R`
-
-### Guides
-- `/Users/tcadman/github-repos/ds-core/dsBaseClient/REFACTOR_GUIDE.md`
-- `/Users/tcadman/github-repos/ds-core/dsBase/.github/pull_request_template`
-
-## Verification
-
-For each batch:
-1. Run server-side unit tests: `cd dsBase && devtools::test(filter = "functionNameDS")`
-2. Run client-side smoke tests: `cd dsBaseClient && devtools::test(filter = "smk-ds.functionName")`
-3. Run `devtools::check(args = '--no-tests')` on both packages
-4. Run full test suite: `devtools::test(filter = "smk-|disc|arg")` to check no regressions
-5. Run perf tests: `PERF_DURATION_SEC=2 devtools::test(filter = "perf-")` to verify no performance regression
-
-## Follow-up: refactor client-side `checkClass`
-
-The client-side helper `checkClass()` currently does two jobs in one: (a) fetches the class of a server-side object and (b) checks the class is consistent across studies. After this refactor, the first job is redundant (the server returns `class` in aggregate results), but the second is still needed by composite dispatchers (`ds.summary` and similar).
-
-Planned cleanup (defer to a dedicated branch):
-
-- Rename `checkClass` → `.checkClass` to mark it internal (matches `.checkClassConsistency`, `.set_datasources`).
-- Split its responsibilities: one helper that fetches class for pre-call routing, one that checks cross-study consistency on a set of classes.
-- Update the remaining callers (`ds.summary` and any others still holding a client-side class pre-fetch).
-
-Not done as part of any single batch because the rename touches callers outside that batch's function set and would break functions not yet refactored. Schedule once all batches are merged — the rename then becomes one small, isolated commit.
diff --git a/_pkgdown.yml b/_pkgdown.yml
index f46c2ebc7..bcec450b1 100644
--- a/_pkgdown.yml
+++ b/_pkgdown.yml
@@ -1,4 +1,5 @@
template:
+ bootstrap: 5
lang: en-GB
params:
bootswatch: simplex
diff --git a/armadillo_azure-pipelines.yml b/armadillo_azure-pipelines.yml
index 2fba0a0a5..edcb04401 100644
--- a/armadillo_azure-pipelines.yml
+++ b/armadillo_azure-pipelines.yml
@@ -134,11 +134,12 @@ jobs:
sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y
sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y
- sudo R -q -e "install.packages(c('devtools','covr'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('fields','meta','metafor','ggplot2','gridExtra','data.table'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('DSI','DSOpal','DSLite'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('MolgenisAuth', 'MolgenisArmadillo', 'DSMolgenisArmadillo'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('DescTools','e1071'), dependencies=TRUE, repos='https://cloud.r-project.org')"
+
+ # Use Posit Public Package Manager, which serves precompiled binary packages
+ # for this Ubuntu release instead of source - so dependencies are downloaded
+ # rather than compiled (the slow part). The HTTPUserAgent option is what makes
+ # the manager hand back binaries.
+ sudo R -q -e "options(repos=c(CRAN=paste0('https://packagemanager.posit.co/cran/__linux__/', system('lsb_release -cs', intern=TRUE), '/latest')), HTTPUserAgent=sprintf('R/%s R (%s)', getRversion(), paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)), Ncpus=4); install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), dependencies=TRUE)"
sudo R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='v6.3.4-dev', dependencies = TRUE)"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 3844edebf..bbd5632b2 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -115,12 +115,12 @@ jobs:
sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y
sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y
- sudo R -q -e "install.packages(c('curl','httr'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('devtools','covr'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('fields','meta','metafor','ggplot2','gridExtra','data.table'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('DSI','DSOpal','DSLite'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('MolgenisAuth', 'MolgenisArmadillo', 'DSMolgenisArmadillo'), dependencies=TRUE, repos='https://cloud.r-project.org')"
- sudo R -q -e "install.packages(c('DescTools','e1071'), dependencies=TRUE, repos='https://cloud.r-project.org')"
+
+ # Use Posit Public Package Manager, which serves precompiled binary packages
+ # for this Ubuntu release instead of source - so dependencies are downloaded
+ # rather than compiled (the slow part). The HTTPUserAgent option is what makes
+ # the manager hand back binaries.
+ sudo R -q -e "options(repos=c(CRAN=paste0('https://packagemanager.posit.co/cran/__linux__/', system('lsb_release -cs', intern=TRUE), '/latest')), HTTPUserAgent=sprintf('R/%s R (%s)', getRversion(), paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)), Ncpus=4); install.packages(c('curl','httr','devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), dependencies=TRUE)"
sudo R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='6.3.4', dependencies = TRUE)"
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 000000000..feeeecd26
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,23 @@
+codecov:
+ notify:
+ # 7 armadillo-dsbase matrix shards each upload their own cobertura.xml
+ # (dsdanger is excluded - see dsBaseClient_test_suite.yaml) - without
+ # this, Codecov has no signal for when all uploads for a commit are in,
+ # and can delay posting codecov/patch by anywhere from minutes to hours.
+ after_n_builds: 7
+
+coverage:
+ status:
+ # Whole-repo coverage trend - informational only, never fails a PR. The
+ # combined figure is still shown in the GitHub Actions job summary
+ # (test-summary job) alongside a link to the full Codecov report.
+ project:
+ default:
+ target: auto
+ informational: true
+ # Coverage of lines actually changed by the PR - this is the enforced
+ # gate, so new/changed code is held to a bar without blocking on the
+ # pre-existing coverage backlog elsewhere in the repo.
+ patch:
+ default:
+ target: 80%
diff --git a/docker-compose_armadillo.yml b/docker-compose_armadillo.yml
index dba71daf8..1cab1c3bb 100644
--- a/docker-compose_armadillo.yml
+++ b/docker-compose_armadillo.yml
@@ -14,6 +14,8 @@ services:
- ./tests/docker/armadillo/standard/data:/data
- ./tests/docker/armadillo/standard/config:/config
- /var/run/docker.sock:/var/run/docker.sock
+ depends_on:
+ - default
default:
hostname: default
diff --git a/docs/404.html b/docs/404.html
index eaa2175ab..1534613f7 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -4,90 +4,70 @@
-
+
Page not found (404) • dsBaseClient
-
-
-
-
-
-
-
+
+
+
+
+
-
-
-
-