diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000000..639bc2b2bf --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +/docs/* \ No newline at end of file diff --git a/.github/linters/.htmlhintrc b/.github/linters/.htmlhintrc new file mode 100644 index 0000000000..3350e566ec --- /dev/null +++ b/.github/linters/.htmlhintrc @@ -0,0 +1,5 @@ +{ + "head-script-disabled": false, + "alt-require": false, + "id-class-value": false +} diff --git a/.github/workflows/.htmlhintrc b/.github/workflows/.htmlhintrc new file mode 100644 index 0000000000..4d1e298c21 --- /dev/null +++ b/.github/workflows/.htmlhintrc @@ -0,0 +1,3 @@ +{ + "head-script-disabled": false +} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000000..1451ef28fb --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,50 @@ +--- +name: Lint + +on: # yamllint disable-line rule:truthy + push: + branches: + - 'master' + pull_request: + branches: + - 'master' + +permissions: {} + +env: + FILTER_REGEX_EXCLUDE: "docs/.*" + +jobs: + build: + name: Lint + runs-on: ubuntu-latest + + permissions: + contents: read + packages: read + # To report GitHub Actions status checks + statuses: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # super-linter needs the full git history to get the + # list of files that changed across commits + fetch-depth: 0 + + - name: Super-linter + uses: super-linter/super-linter@v7.1.0 # x-release-please-version + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HTMLHINT_CONFIG_FILE: .github/linters/.htmlhintrc + VALIDATE_ALL_CODEBASE: false + VALIDATE_ANSIBLE: false + VALIDATE_CHECKOV: false + VALIDATE_JSCPD: false + VALIDATE_LATEX: false + FIX_YAML_PRETTIER: true + VALIDATE_JAVASCRIPT_PRETTIER: false + VALIDATE_JAVASCRIPT_STANDARD: false + VALIDATE_SQLFLUFF: false diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml new file mode 100644 index 0000000000..48f06e057b --- /dev/null +++ b/.github/workflows/static.yml @@ -0,0 +1,47 @@ +# Simple workflow for deploying static content to GitHub Pages +name: Deploy static content to Pages + +on: + # Runs on pushes/marge targeting the default branch + push: + branches: ["master"] + paths: + - 'docs/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Single deploy job since we're just deploying + deploy_static_pages: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload docs only + path: './docs' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ebb461289..6cd1b28477 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout sgn - uses: actions/checkout@v2 + uses: actions/checkout@v5 - name: Run unit tests run: prove --recurse t/unit 2>/dev/null diff --git a/.github/workflows/test_static.yml b/.github/workflows/test_static.yml new file mode 100644 index 0000000000..0bf2da705c --- /dev/null +++ b/.github/workflows/test_static.yml @@ -0,0 +1,45 @@ +# Simple workflow for deploying static content to GitHub Pages +name: Build documentation with R bookmark + +on: + # Runs on pushes/marge targeting the default branch + pull_request: + paths: + - 'docs/r_markdown_docs/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: write + id-token: write + +jobs: + # Single deploy job since we're just deploying + build_and_deploy_static_pages_for_manual: + runs-on: ubuntu-latest + container: + image: bienkowskid/fedora40-r-bookdown + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + - name: Save directory for checkout + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Create gitbook + working-directory: ./docs/r_markdown_docs + run: R -q -e 'bookdown::render_book("index.Rmd", "bookdown::gitbook")' + - name: Create manual pdf + working-directory: ./docs/r_markdown_docs + run: R -q -e 'bookdown::render_book("index.Rmd", "bookdown::pdf_book")' + - name: Commit and push documentation + run: | + date > generated.txt + git config user.name github-actions + git config user.email github-actions@github.com + git add --force --all docs/ + git commit -m "update bookdown docs" + git push diff --git a/.gitignore b/.gitignore index cdf70f82dd..e8fea74ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,7 @@ typescript docs/Gemfile.lock docs/_site/ .DS_Store -sgn.iml \ No newline at end of file +sgn.iml +docs/* +!docs/r_markdown_docs +.idea/ \ No newline at end of file diff --git a/R/DRRC.r b/R/DRRC.r new file mode 100644 index 0000000000..a19b7249a1 --- /dev/null +++ b/R/DRRC.r @@ -0,0 +1,116 @@ + +args=commandArgs(TRUE) + +##args is now a list of character vectors +## First check to see if arguments are passed. +## Then cycle through each element of the list and evaluate the expressions. + +if(length(args)==0){ + print("No arguments supplied.") + ##supply default values + paramfile='' +} else { + for(i in 1:length(args)){ + print(paste("Processing arg ", args[[i]])); + eval(parse(text=args[[i]])) + } +} + +library(reshape2) +library(dplyr) +library(blocksdesign) + +source(paramfile) +## 1) Preparing dataframe +all.clones <- treatments +nTrt <- length(all.clones) +nRep <- nRep +nRows <- nRow +nCols <- nCol +# nCols <- nTrt*nRep/nRows +rowsPerBlock <- nTrt/nCols +colsPerBlock <- nTrt/nRows +superCols <- nCols/colsPerBlock +totalPlots <- nTrt*nRep + +plot_type <- plot_type +plot_start <- plot_start + +blocks = data.frame(block_number = gl(nRep,nTrt), + Cols = gl(superCols,colsPerBlock,totalPlots), + row_number = gl(nRows,nCols,totalPlots), + col_number = gl(nCols,1,totalPlots)) + +## Setting rep number orthogonal to block number +rep_number = as.numeric(blocks$Cols) + + +# treatments = data.frame(treatments =gl(nTrt,1,totalPlots)) +Z=design(all.clones,blocks, searches = 50, weighting=0.5) +fieldBook <- Z$Design + +trialMatrix <- matrix(0,nRows,nCols) + +for(i in 1:nrow(fieldBook)){ + trialMatrix[fieldBook$subRows[i],fieldBook$subCols[i]]<-fieldBook$treatments[i] +} +trialMatrix + +## Adding plot number +colnames(fieldBook)[5] <- "plot_number" + +fieldBook$block_number <- as.integer(fieldBook$block_number) +fieldBook$row_number <- as.integer(fieldBook$row_number) +fieldBook$col_number <- as.integer(fieldBook$col_number) + +# Load dplyr +library(dplyr) + +# Arrange fieldBook by row_number and col_number +fieldBook <- fieldBook %>% arrange(row_number, col_number) +fieldBook$plot_number <- c(1:totalPlots) +fieldBook$plot_id <- c(1:nTrt) + + + +## Number start +## 00101 will be added for NCSU +if(plot_start == "00101"){ + fieldBook$plot_number = paste0(formatC(fieldBook$block_number,width=3,flag="0"), + formatC(fieldBook$plot_id,width=2,flag="0")) +}else if (plot_start == 1001){ + fieldBook$plot_number <- (1000*fieldBook$block_number)+fieldBook$plot_id +}else if (plot_start == 101) { + fieldBook$plot_number <- (100*fieldBook$block_number)+fieldBook$plot_id +} + +cat("plot start is ", plot_start,"\n") +cat("plot type is ", plot_type,"\n") + +plot_type = "serpentine" +## Plot number format +if(plot_type == "serpentine"){ + for(i in 1:nRows){ + if(i%%2==0){ + fieldBook[fieldBook$row_number == i, "plot_number"] <- rev(fieldBook[fieldBook$row_number==i,"plot_number"]) + } + } +} + +fieldBook$rep_number <- rep_number + + +#### create is_a_control +names(fieldBook)[names(fieldBook) == "treatments"] <- "accession_name" +fieldBook <- transform(fieldBook, is_a_control = ifelse(fieldBook$accession_name %in% controls, 1, 0)) + +design <- fieldBook %>% dplyr::select(block_number, rep_number, row_number, col_number, plot_number, accession_name, is_a_control) + +head(design) + +# save result files +basefile <- tools::file_path_sans_ext(paramfile) +outfile = paste(basefile, ".design", sep=""); +sink(outfile) +write.table(design, quote=F, sep='\t', row.names=FALSE) +sink(); \ No newline at end of file diff --git a/R/GCPC.R b/R/GCPC.R index 8ad62afd17..ea9aea9baa 100644 --- a/R/GCPC.R +++ b/R/GCPC.R @@ -15,34 +15,36 @@ # 10. Format the information needed for output. -#Get Arguments -args = commandArgs(trailingOnly=TRUE) +# Get Arguments +args <- commandArgs(trailingOnly = TRUE) if (length(args) < 3) { - stop('Two or more arguments are required.') + stop("Two or more arguments are required.") } -phenotypeFile = args[1] -genotypeFile = args[2] -traits = args[3] -weights = args[4] -userSexes = args[5] -userFixed = args[6] -userRandom = args[7] +phenotypeFile <- args[1] +genotypeFile <- args[2] +traits <- args[3] +weights <- args[4] +userSexes <- args[5] +userFixed <- args[6] +userRandom <- args[7] -#userSexes = as.vector(userSexes) +# userSexes = as.vector(userSexes) -#L = length(userSexes) -#if (L==1 && userSexes[1]!="") {write('PLANT SEX is empty: ', stderr())} -write(paste('PLANT SEX CVTERM: |', userSexes, "|"), stderr()) +# L = length(userSexes) +# if (L==1 && userSexes[1]!="") {write('PLANT SEX is empty: ', stderr())} +write(paste("PLANT SEX CVTERM: |", userSexes, "|"), stderr()) ################################################################################ # 1. Load software needed ################################################################################ - +library(dplyr) +library(tidyr) library(sommer) library(AGHmatrix) library(VariantAnnotation) # Bioconductor package library(tools) +# library(rstatix) Rcpp::sourceCpp("/home/production/cxgn/QuantGenResources/CalcCrossMeans.cpp") # this is called CalcCrossMean.cpp on Github @@ -57,20 +59,20 @@ Rcpp::sourceCpp("/home/production/cxgn/QuantGenResources/CalcCrossMeans.cpp") # # a. Define path with internal YamBase instructions such that the object 'userGeno' # is defined as a VCF file of genotypes. -#userGeno <- path +# userGeno <- path # b. Define path2 with internal YamBase instructions such that the object 'userPheno' # is defined as the phenotype file. -#userPheno <- path2 -#write(paste("READING PHENOTYPEFILE: ",phenotypeFile), stderr()) -userPheno <- read.delim(phenotypeFile, header = TRUE, sep="\t", fill=TRUE) #testing only -#write(colnames(userPheno), stderr()) -#write(summary(userPheno), stderr()) -##userPheno <- userPheno[userPheno$Trial == "SCG", ] #testing only-- needs to replaced with 2-stage +# userPheno <- path2 +# write(paste("READING PHENOTYPEFILE: ",phenotypeFile), stderr()) +userPheno <- read.delim(phenotypeFile, header = TRUE, sep = "\t", fill = TRUE) # testing only +# write(colnames(userPheno), stderr()) +# write(summary(userPheno), stderr()) +## userPheno <- userPheno[userPheno$Trial == "SCG", ] #testing only-- needs to replaced with 2-stage -#write("DONE WITH PHENOTYPEFILE"), stderr()) +# write("DONE WITH PHENOTYPEFILE"), stderr()) # c. The user should be able to select their fixed variables from a menu # of the column names of the userPheno object. The possible interaction terms @@ -79,9 +81,9 @@ userPheno <- read.delim(phenotypeFile, header = TRUE, sep="\t", fill=TRUE) #test # besides f are requested. # f is automatically included as a fixed effect- a note to the user would be good. -#userFixed <- c() -#userFixed <- c("studyYear") # for testing only -userFixed <-unlist(strsplit(userFixed, split=",", fixed=T)) +# userFixed <- c() +# userFixed <- c("studyYear") # for testing only +userFixed <- unlist(strsplit(userFixed, split = ",", fixed = T)) # d. The user should be able to select their random variables from a menu @@ -89,16 +91,16 @@ userFixed <-unlist(strsplit(userFixed, split=",", fixed=T)) # also need to be shown somehow. Then, those strings should be passed # to this vector, 'userRandom'. -#userRandom <- c() -#userRandom <- "blockNumber" # for testing only -userRandom <-unlist(strsplit(userRandom, split=",", fixed=T)) +# userRandom <- c() +# userRandom <- "blockNumber" # for testing only +userRandom <- unlist(strsplit(userRandom, split = ",", fixed = T)) # e. The user should be able to indicate which of the userPheno column names # represents individual genotypes identically as they are represented in the VCF # column names. No check to ensure matching at this stage. This single string # should be passed to this vector, userID. -#userID <- c() +# userID <- c() userID <- "germplasmName" # for testing only @@ -123,25 +125,25 @@ userPloidy <- 2 # for testing only # to this vector, 'userResponse'. write(paste("TRAIT STRING:", traits), stderr()) -#userResponse <- c() -#userResponse <- c("YIELD", "DMC", "OXBI") # for testing only -userResponse <- unlist(strsplit(traits, split=",", fixed=T)) +# userResponse <- c() +# userResponse <- c("YIELD", "DMC", "OXBI") # for testing only +userResponse <- unlist(strsplit(traits, split = ",", fixed = T)) -write(paste("USER RESPONSE", userResponse),stderr()) +write(paste("USER RESPONSE", userResponse), stderr()) write(paste("first element: ", userResponse[1]), stderr()) # h. The user must indicate weights for each response. The order of the vector # of response weights must match the order of the responses in userResponse. -#userWeights <- c() -#userWeights <- c(1, 0.8, 0.2) # for YIELD, DMC, and OXBI respectively; for testing only -userWeights <- as.numeric(unlist(strsplit(weights, split=",", fixed=T))) +# userWeights <- c() +# userWeights <- c(1, 0.8, 0.2) # for YIELD, DMC, and OXBI respectively; for testing only +userWeights <- as.numeric(unlist(strsplit(weights, split = ",", fixed = T))) write(paste("WEIGHTS", userWeights), stderr()) # i. The user can indicate the number of crosses they wish to output. # The maximum possible is a full diallel. -#userNCrosses <- c() +# userNCrosses <- c() userNCrosses <- 100 # for testing only @@ -152,9 +154,9 @@ userNCrosses <- 100 # for testing only # then userSexes should be set to NA. -#userSexes <- c() -#userSexes <- "Sex" # for testing only -#userPheno$Sex <- sample(c("M", "F"), size = nrow(userPheno), replace = TRUE, prob = c(0.7, 0.3)) # for testing only +# userSexes <- c() +# userSexes <- "Sex" # for testing only +# userPheno$Sex <- sample(c("M", "F"), size = nrow(userPheno), replace = TRUE, prob = c(0.7, 0.3)) # for testing only # Please note that for the test above, sex is sampled randomly for each entry, so the same accession can have # different sexes. This does not matter for the code or testing. @@ -181,34 +183,30 @@ userNCrosses <- 100 # for testing only # monomorphic or biallelic sites which could be communicated through the GUI. # It's also possible to filter them here. -if (file_ext(genotypeFile) == 'vcf') { - write(paste("READING VARIANT FILE ", genotypeFile), stderr()) - # Import VCF with VariantAnnotation package and extract matrix of dosages - myVCF <- readVcf(genotypeFile) - #G <- t(geno(myVCF)$DS) # Individual in row, genotype in column - mat <- genotypeToSnpMatrix(myVCF) - #G <- t(geno(myVCF)$DS) # Individual in row, genotype in column - G <- as(mat$genotypes, "numeric") - G <- G[,colSums(is.na(G)) 2){ -# Digenic dominance -C_matrix = matrix(length(combn(userPloidy, 2)) / 2, - nrow = nrow(t(G)), - ncol = ncol(t(G))) - -Ploidy_matrix = matrix(userPloidy, - nrow = nrow(t(G)), - ncol = ncol(t(G))) - -Q = (allele_freq^2 * C_matrix) - - (Ploidy_matrix - 1) * allele_freq * t(G) + - 0.5 * t(G) * (t(G) - 1) - -Dnum = crossprod(Q) -denomDom = sum(C_matrix[ ,1] * allele_freq^2 * (1 - allele_freq)^2) -D = Dnum/denomDom +if (userPloidy > 2) { + # Digenic dominance + C_matrix <- matrix(length(combn(userPloidy, 2)) / 2, + nrow = nrow(t(G)), + ncol = ncol(t(G)) + ) + + Ploidy_matrix <- matrix(userPloidy, + nrow = nrow(t(G)), + ncol = ncol(t(G)) + ) + + Q <- (allele_freq^2 * C_matrix) - + (Ploidy_matrix - 1) * allele_freq * t(G) + + 0.5 * t(G) * (t(G) - 1) + + Dnum <- crossprod(Q) + denomDom <- sum(C_matrix[, 1] * allele_freq^2 * (1 - allele_freq)^2) + D <- Dnum / denomDom } @@ -295,37 +295,37 @@ D = Dnum/denomDom # 5. Process the phenotypic data. ################################################################################ -#write(summary(userPheno), stderr()) +# write(summary(userPheno), stderr()) # a. Paste f into the phenotype dataframe write("processing phenotypic data...", stderr()) -userPheno$f <- f[as.character(userPheno[ , userID])] +userPheno$f <- f[as.character(userPheno[, userID])] -#write(summary(userPheno), stderr()) +# write(summary(userPheno), stderr()) # b. Scale the response variables. write("processing phenotypic data... scaling...", stderr()) write(paste("USER RESPONSE LENGTH = ", length(userResponse)), stderr()) -for(i in 1:length(userResponse)){ +for (i in 1:length(userResponse)) { write(paste("working on user response ", userResponse[i]), stderr()) - userPheno[ , userResponse[i]] <- (userPheno[ , userResponse[i]] - mean(userPheno[ , userResponse[i]], na.rm = TRUE)) / sd(userPheno[ , userResponse[i]], na.rm = TRUE) + userPheno[, userResponse[i]] <- (userPheno[, userResponse[i]] - mean(userPheno[, userResponse[i]], na.rm = TRUE)) / sd(userPheno[, userResponse[i]], na.rm = TRUE) } -write(paste("accession count: ", length(userPheno[ , userID])), stderr()) +write(paste("accession count: ", length(userPheno[, userID])), stderr()) write("processing phenotypic data... adding dominance effects", stderr()) # c. Paste in a second ID column for the dominance effects. -#write(summary(userPheno), stderr()) +# write(summary(userPheno), stderr()) -dominanceEffectCol = paste(userID, "2", sep="") +dominanceEffectCol <- paste(userID, "2", sep = "") write(paste("NEW COL NAME: ", dominanceEffectCol), stderr()) -write(paste("USER_ID COLUMN: ", userPheno[ , userID]), stderr()); - - -userPheno[ , dominanceEffectCol] <- userPheno[ , userID] +write(paste("USER_ID COLUMN: ", userPheno[, userID]), stderr()) +userPheno[, dominanceEffectCol] <- userPheno[, userID] -write(paste("USER PHENO userID2 COL", userPheno[ , dominanceEffectCol]), stderr()) +write(paste("USER PHENO userID2 COL", userPheno[, dominanceEffectCol]), stderr()) +uniq <- length(sapply(lapply(userPheno, unique), length)) +write(paste("UNIQUE", uniq), stderr()) # Additional steps could be added here to remove outliers etc. @@ -338,23 +338,21 @@ write(paste("USER PHENO userID2 COL", userPheno[ , dominanceEffectCol]), stderr( # 6. Fit the mixed models in sommer. ################################################################################ -write("Fit mixed model in sommer", stderr()); - +write("Fit mixed model in sommer", stderr()) # 6a. Make a list to save the models. userModels <- list() -for(i in 1:length(userResponse)){ - +for (i in 1:length(userResponse)) { write(paste("User response: ", userResponse[i]), stderr()) # check if fixed effects besides f are requested, then paste together # response variable and fixed effects - if(!is.na(userFixed[1])){ - fixedEff <- paste(userFixed, collapse = " + ") - fixedEff <- paste(fixedEff, "f", sep = " + ") - fixedArg <- paste(userResponse[i], " ~ ", fixedEff, sep = "") + if (!is.na(userFixed[1])) { + fixedEff <- paste(userFixed, collapse = " + ") + fixedEff <- paste(fixedEff, "f", sep = " + ") + fixedArg <- paste(userResponse[i], " ~ ", fixedEff, sep = "") } - if(is.na(userFixed[1])){ + if (is.na(userFixed[1])) { fixedArg <- paste(userResponse[i], " ~ ", "f") } @@ -364,14 +362,14 @@ for(i in 1:length(userResponse)){ write("Generate formula...", stderr()) - if(!is.na(userRandom[1])){ + if (!is.na(userRandom[1])) { randEff <- paste(userRandom, collapse = " + ") ID2 <- paste(userID, 2, sep = "") randEff2 <- paste("~vsr(", userID, ", Gu = A) + vsr(", ID2, ", Gu = D)", sep = "") randArg <- paste(randEff2, randEff, sep = " + ") } - if(is.na(userRandom[1])){ - ID2 <- paste(userID, 2, sep ="") + if (is.na(userRandom[1])) { + ID2 <- paste(userID, 2, sep = "") randArg <- paste("~vsr(", userID, ", Gu = A) + vsr(", ID2, ", Gu = D)", sep = "") } @@ -380,16 +378,18 @@ for(i in 1:length(userResponse)){ # write(paste("USER PHENO:", userPheno), stderr()) # write(paste("COLNAMES: ", colnames(userPheno)), stderr()) # fit the mixed GBLUP model - myMod <- mmer(fixed = as.formula(fixedArg), - random = as.formula(randArg), - rcov = ~units, - getPEV = FALSE, - data = userPheno) + myMod <- mmer( + fixed = as.formula(fixedArg), + random = as.formula(randArg), + rcov = ~units, + getPEV = FALSE, + data = userPheno + ) # save the fit model - write(paste("I = ", i), stderr()); + write(paste("I = ", i), stderr()) userModels[[i]] <- myMod } @@ -406,7 +406,7 @@ for(i in 1:length(userResponse)){ # a. Get the matrices and inverses needed # This is not correct for polyploids yet. A.G <- G - (userPloidy / 2) # this is the additive genotype matrix (coded -1 0 1 for diploids) -D.G <- 1 - abs(A.G) # this is the dominance genotype matrix (coded 0 1 0 for diploids) +D.G <- 1 - abs(A.G) # this is the dominance genotype matrix (coded 0 1 0 for diploids) A.T <- A.G %*% t(A.G) ## additive genotype matrix @@ -425,8 +425,7 @@ write("backsolve marker effects...", stderr()) userAddEff <- list() # save them in order userDomEff <- list() # save them in order -for(i in 1:length(userModels)){ - +for (i in 1:length(userModels)) { myMod <- userModels[[i]] # get the additive and dominance effects out of the sommer list @@ -438,7 +437,7 @@ for(i in 1:length(userModels)){ # backsolve addEff <- A.TTinv %*% matrix(subModA[colnames(A.TTinv)], ncol = 1) # these must be reordered to match A.TTinv - domEff <- D.TTinv %*% matrix(subModD[colnames(D.TTinv)], ncol=1) # these must be reordered to match D.TTinv + domEff <- D.TTinv %*% matrix(subModD[colnames(D.TTinv)], ncol = 1) # these must be reordered to match D.TTinv # add f coefficient back into the dominance effects subModf <- myMod$Beta @@ -464,17 +463,17 @@ write("weight marker effects...", stderr()) ai <- 0 di <- 0 -for(i in 1:length(userWeights)){ - #write(paste("USER ADD EFF : ", userAddEff[[i]]), stderr()) +for (i in 1:length(userWeights)) { + # write(paste("USER ADD EFF : ", userAddEff[[i]]), stderr()) # write(paste("USER DOM EFF : ", userDomEff[[i]]), stderr()) - #write(paste("USER WEIGHT : ", userWeights[i]), stderr()) + # write(paste("USER WEIGHT : ", userWeights[i]), stderr()) - ai <- ai + userAddEff[[i]] * userWeights[i] + ai <- ai + userAddEff[[i]] * userWeights[i] - write("DONE WITH ADDITIVE EFFECTS!\n", stderr()) + write("DONE WITH ADDITIVE EFFECTS!\n", stderr()) di <- di + userDomEff[[i]] * userWeights[i] - write("DONE WITH DOM EFFECTS!\n", stderr()) + write("DONE WITH DOM EFFECTS!\n", stderr()) } @@ -493,17 +492,19 @@ for(i in 1:length(userWeights)){ write("Predict crosses...", stderr()) -GP <- G[rownames(G) %in% userPheno[ , userID], ] +GP <- G[rownames(G) %in% userPheno[, userID], ] print("GP:") print(head(GP)) write("calcCrossMean...", stderr()) -crossPlan <- calcCrossMean(GP, - ai, - di, - userPloidy) +crossPlan <- calcCrossMean( + GP, + ai, + di, + userPloidy +) write("Done with calcCrossMean!!!!!!", stderr()) @@ -517,25 +518,25 @@ write("Done with calcCrossMean!!!!!!", stderr()) # Add option to remove crosses with incompatible sexes. -#hash <- new.env(hash = TRUE, parent = emptyenv(), size = 100L) +# hash <- new.env(hash = TRUE, parent = emptyenv(), size = 100L) -#assign_hash(userPheno$germplasmName, userPheno$userSexes, hash) +# assign_hash(userPheno$germplasmName, userPheno$userSexes, hash) -if(userSexes!=""){ # "plant sex estimation 0-4" -#!is.na(userSexes) +if (userSexes != "") { # "plant sex estimation 0-4" + # !is.na(userSexes) && !is.na(sd(userPheno[, userSexes])) - write(paste("userSexes", head(userSexes)), stderr()) + write(paste("userSexes", sd(userPheno[, userSexes])), stderr()) # Reformat the cross plan crossPlan <- as.data.frame(crossPlan) write(paste("CROSSPLAN = ", head(crossPlan)), stderr()) - crossPlan <- crossPlan[order(crossPlan[,3], decreasing = TRUE), ] # orders the plan by predicted merit - crossPlan[ ,1] <- rownames(GP)[crossPlan[ ,1]] # replaces internal ID with genotye file ID - crossPlan[ ,2] <- rownames(GP)[crossPlan[ ,2]] # replaces internal ID with genotye file ID + crossPlan <- crossPlan[order(crossPlan[, 3], decreasing = TRUE), ] # orders the plan by predicted merit + crossPlan[, 1] <- rownames(GP)[crossPlan[, 1]] # replaces internal ID with genotye file ID + crossPlan[, 2] <- rownames(GP)[crossPlan[, 2]] # replaces internal ID with genotye file ID colnames(crossPlan) <- c("Parent1", "Parent2", "CrossPredictedMerit") - write(paste("CROSSPLAN REPLACED = ", head(crossPlan)), stderr()); + write(paste("CROSSPLAN REPLACED = ", head(crossPlan)), stderr()) # Look up the parent sexes and subset crossPlan$P1Sex <- userPheno[match(crossPlan$Parent1, userPheno$germplasmName), userSexes] # get sexes ordered by Parent1 @@ -545,39 +546,86 @@ if(userSexes!=""){ # "plant sex estimation 0-4" crossPlan$P2Sex <- userPheno[match(crossPlan$Parent2, userPheno$germplasmName), userSexes] # get sexes ordered by Parent2 write(paste("PARENTS2 ", head(crossPlan)), stderr()) + col_repl <- c("P1Sex", "P2Sex") + crossPlan %>% filter(P1Sex == 0 | P2Sex == 0) # remove the 0s + crossPlan %>% filter(P1Sex == 1 & P2Sex == 1) # remove same sex crosses with score of 1 + crossPlan %>% filter(P1Sex == 2 & P2Sex == 2) # remove same sex crosses with score of 2 - crossPlan <- crossPlan[!(crossPlan$P1Sex==0 | crossPlan$P2Sex==0),] #remove the 0s - crossPlan <- crossPlan[!(crossPlan$P1Sex==1 & crossPlan$P2Sex==1),] #remove same sex crosses with score of 1 - crossPlan <- crossPlan[!(crossPlan$P1Sex==2 & crossPlan$P2Sex==2),] #remove same sex crosses with score of 2 + write(paste("CROSSPLAN FILTERED = ", crossPlan), stderr()) + # crossPlan <- crossPlan[crossPlan$P1Sex != crossPlan$P2Sex, ] # remove crosses with same-sex parents - write(paste("CROSSPLAN FILTERED = ", head(crossPlan)), stderr()) - #crossPlan <- crossPlan[crossPlan$P1Sex != crossPlan$P2Sex, ] # remove crosses with same-sex parents + ## replace plant sex numbers to male, female etc - # subset the number of crosses the user wishes to output - crossPlan[1:userNCrosses, ] - finalcrosses=crossPlan[1:userNCrosses, ] - outputFile= paste(phenotypeFile, ".out", sep="") + crossPlan[col_repl] <- sapply(crossPlan[col_repl], function(x) replace(x, x %in% "NA", "NA")) + crossPlan[col_repl] <- sapply(crossPlan[col_repl], function(x) replace(x, x %in% 1, "Male")) + crossPlan[col_repl] <- sapply(crossPlan[col_repl], function(x) replace(x, x %in% 2, "Female")) + crossPlan[col_repl] <- sapply(crossPlan[col_repl], function(x) replace(x, x %in% 3, "Monoecious male (m>f)")) + crossPlan[col_repl] <- sapply(crossPlan[col_repl], function(x) replace(x, x %in% 4, "Monoecious female(f>m)")) - write.csv(finalcrosses, outputFile) + # ** summary statistics for the cross prediction merit + avg <- round(mean(crossPlan$CrossPredictedMerit), digits = 3) + max <- round(max(crossPlan$CrossPredictedMerit), digits = 3) + min <- round(min(crossPlan$CrossPredictedMerit), digits = 3) + std <- round(sd(crossPlan$CrossPredictedMerit), digits = 3) + leng <- length(crossPlan$CrossPredictedMerit) -} + ## histogram + histogra <- paste(phenotypeFile, ".png", sep = "") + png(file = histogra, width = 600, height = 350) + hist(crossPlan$CrossPredictedMerit, xlab = "Cross Predicted Merit", main = "Distribution") + mtext(paste("Mean =", avg), side = 3, adj = 1, line = 0) + mtext(paste("Standard Deviation = ", std), side = 3, adj = 1, line = -1) + mtext(paste("Range = (", min, " to ", max, ")"), side = 3, adj = 1, line = -2) + mtext(paste("No. of predictions = ", leng), side = 3, adj = 1, line = -3) + dev.off() -if(userSexes==""){ + # subset the number of crosses the user wishes to output + if (nrow(crossPlan)<100) { + finalcrosses = crossPlan + } else { + crossPlan[1:userNCrosses, ] + finalcrosses=crossPlan[1:userNCrosses, ] + } + outputFile <- paste(phenotypeFile, ".out", sep = "") + + write.csv(finalcrosses, outputFile) +} else { # only subset the number of crosses the user wishes to output crossPlan <- as.data.frame(crossPlan) - - crossPlan <- crossPlan[order(crossPlan[,3], decreasing = TRUE), ] # orders the plan by predicted merit - crossPlan[ ,1] <- rownames(GP)[crossPlan[ ,1]] # replaces internal ID with genotye file ID - crossPlan[ ,2] <- rownames(GP)[crossPlan[ ,2]] # replaces internal ID with genotye file ID + crossPlan <- na.omit(crossPlan) + crossPlan <- crossPlan[order(crossPlan[, 3], decreasing = TRUE), ] # orders the plan by predicted merit + crossPlan[, 1] <- rownames(GP)[crossPlan[, 1]] # replaces internal ID with genotye file ID + crossPlan[, 2] <- rownames(GP)[crossPlan[, 2]] # replaces internal ID with genotye file ID colnames(crossPlan) <- c("Parent1", "Parent2", "CrossPredictedMerit") - crossPlan[1:userNCrosses, ] - finalcrosses=crossPlan[1:userNCrosses, ] - outputFile= paste(phenotypeFile, ".out", sep="") + # get summary statistics for the cross prediction merit + avg <- round(mean(crossPlan$CrossPredictedMerit), digits = 3) + max <- round(max(crossPlan$CrossPredictedMerit), digits = 3) + min <- round(min(crossPlan$CrossPredictedMerit), digits = 3) + std <- round(sd(crossPlan$CrossPredictedMerit), digits = 3) + leng <- length(crossPlan$CrossPredictedMerit) + + + ## histogram + histogra <- paste(phenotypeFile, ".png", sep = "") + png(file = histogra, width = 600, height = 350) + hist(crossPlan$CrossPredictedMerit, xlab = "Cross Predicted Merit", main = "Distribution") + mtext(paste("Mean =", avg), side = 3, adj = 1, line = 0) + mtext(paste("Standard Deviation = ", std), side = 3, adj = 1, line = -1) + mtext(paste("Range = (", min, " to ", max, ")"), side = 3, adj = 1, line = -2) + mtext(paste("No. of predictions = ", leng), side = 3, adj = 1, line = -3) + dev.off() + + ## save the best 100 predictions + if (nrow(crossPlan)<100) { + finalcrosses = crossPlan + } else { + crossPlan[1:userNCrosses, ] + finalcrosses=crossPlan[1:userNCrosses, ] + } + outputFile <- paste(phenotypeFile, ".out", sep = "") write.csv(finalcrosses, outputFile) - - } diff --git a/R/Nirs/nirs_upload_filter_aggregate.R b/R/Nirs/nirs_upload_filter_aggregate.R index 8f3d757828..6f7beefdf1 100644 --- a/R/Nirs/nirs_upload_filter_aggregate.R +++ b/R/Nirs/nirs_upload_filter_aggregate.R @@ -42,6 +42,11 @@ window.increase.global <<- TRUE; window.size.global <<- 100; while (window.increase.global) { + if (window.size.global > 5000) { + message("Window size exceeded 5000. Exiting loop.") + spec.plot <- NULL + break + } tryCatch( expr = { spec.plot <- plot_spectra(raw.spectra.temp, num.col.before.spectra = 3, window.size = window.size.global) @@ -110,4 +115,4 @@ agg.spectra %>% rename(device_type_rename = Group.2) %>% dplyr::select(sample_name, starts_with("nirs_spectra")) %>% rename_at(vars(starts_with("nirs_spectra")), ~str_replace(., "nirs_spectra.", "")) %>% - write.csv(x=., file = args[2], row.names=FALSE) \ No newline at end of file + write.csv(x=., file = args[2], row.names=FALSE) diff --git a/R/Spatial_Correction.R b/R/Spatial_Correction.R new file mode 100644 index 0000000000..b3cf88e404 --- /dev/null +++ b/R/Spatial_Correction.R @@ -0,0 +1,170 @@ +# ################################################################################ +# # Modeling Spatial Variation +# ################################################################################ + +# # There are ten main steps to this protocol: +# # 1. Load the software needed. +# # 2. Declare user-supplied variables. +# # 3. Process the phenotypic data. +# # 4. Fit the two models with and without 2D Spline model in sommer +# # 5. Format the information needed for output. + + +# ################################################################################ +# # 1. Load software needed +# ################################################################################ + +library(SpATS) +library(spdep) +library(dplyr) + +# ################################################################################ +# # 2. Declare user-supplied variables. +# ################################################################################ +# Get Arguments +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Two or more arguments are required.") +} +phenotypeFile <- args[1] +spatialFile <- args[2] +row_col_as_rc <- args[3] +genotype_as_random <- FALSE +if (args[4] == 1) { + genotype_as_random <- TRUE +} +nseg_degree <- as.numeric(args[5]) +print(nseg_degree) +spatialHeaders <- args[6] + + +# ################################################################################ +# # 3. Process the phenotypic data. +# ################################################################################ +# # read in the phenotypic data + +userPheno <- read.delim(phenotypeFile, header = TRUE, sep = "\t", fill = TRUE, check.names = FALSE) +spatialPheno <- read.delim(spatialFile, header = TRUE, sep = "\t", fill = TRUE) +colnames(spatialPheno) <- c("Trait", "Data.Quality", "Moran.P.Value", "Correction.Needed") +traits <- spatialPheno$Trait[spatialPheno$Correction.Needed == "YES"] +# The user should be able to select their response variables from a drop-down menu +# of the column names of the userPheno object. Then, those strings should be passed +# to this vector, 'userResponse'. +# write(colnames(userPheno), stderr()) + +userResponse <- unlist(strsplit(traits, split = ",", fixed = T)) +userResponse <- userResponse[!userResponse == "notes"] # x[ !x == 'A'] # remove notes from userResponse +rownames(userPheno) <- userPheno$observationUnitName +userPheno$germplasmName <- as.factor(userPheno$germplasmName) +userPheno$R <- as.factor(userPheno$rowNumber) +userPheno$C <- as.factor(userPheno$colNumber) + +output <- data.frame( + observationUnitName = userPheno$observationUnitName, + germplasmName = userPheno$germplasmName, + rowNumber = userPheno$rowNumber, + colNumber = userPheno$colNumber, + replicate = userPheno$replicate, + blockNumber = userPheno$blockNumber, + plotNumber = userPheno$plotNumber +) + +round_to_even <- function(n) { + ifelse(n %% 2 == 0, round(n), round(n / 2) * 2) +} + +moran_outfile <- paste(phenotypeFile, ".moran", sep="") +model_string_outfile <- paste(phenotypeFile, ".model_string", sep="") + +cat("", sep="\t", file = moran_outfile, append = FALSE) #clear this file if not already empty +cat("", sep="\t", file = model_string_outfile, append = FALSE) + +for (trait in userResponse) { + spatial_model <- NULL + spatial_call <- NULL + if (row_col_as_rc == 1) { + spatial_model <- SpATS( + response = trait, + spatial = ~ PSANOVA(colNumber, rowNumber, + nseg = c(round_to_even(max(userPheno$colNumber, na.rm = TRUE) * nseg_degree), round_to_even(max(userPheno$rowNumber, na.rm = TRUE) * nseg_degree)), + degree = c(3,3), + nest.div = 2), + genotype = "germplasmName", + genotype.as.random = genotype_as_random, + random = ~ R + C , + fixed = NULL, + data = userPheno, + control = list(tolerance = 1e-03, monitoring = 1) + ) + spatial_call <- paste(sep="", "SpATS(response = trait, spatial = ~ PSANOVA(colNumber, rowNumber, nseg = (", + round_to_even(max(userPheno$colNumber, na.rm = TRUE) * nseg_degree),",", round_to_even(max(userPheno$rowNumber, na.rm = TRUE) * nseg_degree), "), degree = (3,3), nest.div = 2, genotype.as.random = ", + genotype_as_random, ", random = ~ R + C, fixed = NULL, control = list(tolerance = 1e-03, monitoring = 1)") + } else { + spatial_model <- SpATS( + response = trait, + spatial = ~ PSANOVA(colNumber, rowNumber, + nseg = c(round_to_even(max(userPheno$colNumber) * nseg_degree), round_to_even(max(userPheno$rowNumber) * nseg_degree)), + degree = c(3,3), + nest.div = 2), + genotype = "germplasmName", + genotype.as.random = genotype_as_random, + # random = ~ R + C , + data = userPheno, + fixed = NULL, + control = list(tolerance = 1e-03, monitoring = 1) + ) + spatial_call <- paste(sep="", "SpATS(response = trait, spatial = ~ PSANOVA(colNumber, rowNumber, nseg = (", + round_to_even(max(userPheno$colNumber, na.rm = TRUE) * nseg_degree),",", round_to_even(max(userPheno$rowNumber, na.rm = TRUE) * nseg_degree), "), degree = (3,3), nest.div = 2, genotype.as.random = ", + genotype_as_random, ", fixed = NULL, control = list(tolerance = 1e-03, monitoring = 1)") + } + + summary(spatial_model) + + cat(spatial_call, file = model_string_outfile, append = FALSE) + + residuals <- as.data.frame(residuals(spatial_model)) + acc_blues <- as.data.frame(predict.SpATS(spatial_model, which='germplasmName')) + + userPheno$residuals <- residuals[["residuals(spatial_model)"]] + + plot_adjusted_vals <- merge(userPheno, acc_blues, by = 'germplasmName', sort = FALSE, all.x = TRUE) + rownames(plot_adjusted_vals) <- plot_adjusted_vals$observationUnitName + + coordinates <- userPheno[, c("rowNumber", "colNumber"), drop = FALSE] + k <- 3 + kn <- knearneigh(coordinates, k = k) + nb <- knn2nb(kn) + weights <- nb2listw(nb) + moran <- NULL + tryCatch({ + moran <- moran.test(plot_adjusted_vals[[trait]] + plot_adjusted_vals$residuals, weights, na.action = na.exclude ) + }, + error = function(e) { #This happens when there is missing data. + moran$p.value <- NaN + }, + finally = { + print(paste("Moran p-value for trait ",trait, " : ", moran$p.value)) #sanity check, spatial autocorrelation should be gone, residuals should not show a spatial pattern + cat(trait, moran$p.value, sep="\t", file = moran_outfile, append = TRUE) + cat("\n", sep="\t", file = moran_outfile, append = TRUE) + } + ) + domain <- NULL + range <- list(min = min(userPheno[[trait]], na.rm = TRUE), max = max(userPheno[[trait]], na.rm = TRUE)) + if (range$min >= 0 & range$max > 0) { #strictly positive trait vals + domain <- list(min = 0, max = Inf) + } else if (range$min < 0 & range$max > 0) { # pos or neg trait vals + domain <- list(min = -Inf, max = Inf) + } else { # strictly neg trait vals + domain <- list(min = -Inf, max = 0) + } + + output[[trait]] <- userPheno[[trait]] + output[[paste(trait, "_spatially_corrected", sep = "")]] <- plot_adjusted_vals[[trait]] + plot_adjusted_vals$residuals + output[[paste(trait, "_spatially_corrected", sep = "")]][output[[paste(trait, "_spatially_corrected", sep = "")]] < domain$min] <- domain$min #prevent out of bounds values + output[[paste(trait, "_spatially_corrected", sep = "")]][output[[paste(trait, "_spatially_corrected", sep = "")]] > domain$max] <- domain$max + output[[paste(trait, "_spatial_adjustment", sep = "")]] <- output[[paste(trait, "_spatially_corrected", sep = "")]] - output[[trait]] +} + +outfile <- paste(phenotypeFile, ".spatially_corrected", sep="") + +write.table(output, file = outfile, quote = FALSE, sep = "\t", col.names = TRUE, row.names = FALSE) diff --git a/R/dataset/rosner_test.R b/R/dataset/rosner_test.R new file mode 100644 index 0000000000..2563814695 --- /dev/null +++ b/R/dataset/rosner_test.R @@ -0,0 +1,39 @@ +# rosners_test + +library('EnvStats') + +##### Get data ##### +args = commandArgs(trailingOnly = TRUE) + +# (input_file_name, output_file_name, trait_name) +input_file_name <- args[1] +print(input_file_name) +output_file_name <- args[2] +print(output_file_name) +trait_name <- args[2] + +# errorMessages <- c() + +table <- read.csv(input_file_name) + +##### remove NA ##### +table <- table[rowSums(is.na(table)) != ncol(table), ] + +outliers_number = 3 + +##### Make test ##### +test <- rosnerTest(table[, 1], k = outliers_number) + +# table[test$all.stats$Obs.Num, 2] + +##### Default for test k = 3 loop if more outliers then 3 - it should return table with one extra row with first non outlier value ##### +while (test$n.outliers == outliers_number) { + outliers_number = outliers_number + 1 + test <- rosnerTest(table[, 1], k = outliers_number) +} + +# change observation number in data frame to phenotype id +test$all.stats$Obs.Num <- table[test$all.stats$Obs.Num, 2] + +# write csv file without first row +write.csv(file = output_file_name, test$all.stats, row.names = FALSE) \ No newline at end of file diff --git a/R/mixed_models.R b/R/mixed_models.R index 8cac811bd5..e5ca5a8e1c 100644 --- a/R/mixed_models.R +++ b/R/mixed_models.R @@ -163,4 +163,4 @@ if (genotypeEffectType=="random") { sink(outfile_adjblues) write.table(adjustedBLUE, quote=F , sep='\t', row.names=FALSE) sink(); -} +} \ No newline at end of file diff --git a/R/mixed_models_sommer.R b/R/mixed_models_sommer.R index 558c79eb51..ee36d55e0d 100644 --- a/R/mixed_models_sommer.R +++ b/R/mixed_models_sommer.R @@ -76,33 +76,39 @@ for(i in 1:length(trait)){ if (genotypeEffectType=="random") { mixmodel = mmer(as.formula(fixed_model), random = as.formula(random_model), rcov = ~ units, data=pd) + # print(paste("MIXED MODEL: ", mixmodel)) + # varcomp<- summary(mixmodel)$varcomp + # print(varcomp) - varcomp<- summary(mixmodel)$varcomp - print(varcomp) + # print("---------") - print("---------") - - print("---------") + # print("---------") ##BLUPS res <- randef(mixmodel) ##obtain the blups BLUP <- as.data.frame(res) - #print(BLUP) + BLUP <- tibble::rownames_to_column(BLUP, var = 'accession name') + print(BLUP) ##ajusted means - p0 = predict.mmer(mixmodel, classify="germplasmName") ##runs the prediction + p0 = predict.mmer(mixmodel, D="germplasmName") ##runs the prediction summary(p0) adj = p0$pvals ##obtains the predictions adjusted_means = as.data.frame(adj) #print(paste("adj", adj)) - print(adjusted_means) + # print(adjusted_means) } else { - + if (random_model!="") { mixmodel = mmer(as.formula(fixed_model), random = as.formula(random_model), rcov = ~ units, data=pd) - varcomp<-summary(mixmodel)$varcomp + } else { + mixmodel = mmer(as.formula(fixed_model), rcov = ~ units, data=pd) + } + + varcomp<-summary(mixmodel)$varcomp + print(paste("MIXED MODEL: ", mixmodel)) #Computing fixed effects blue = summary(mixmodel)$beta @@ -111,7 +117,7 @@ for(i in 1:length(trait)){ #BLUE = merge(x = BLUE, y = fixedeff, by="germplasmName", all=TRUE) # compute adjusted blues - p0 = predict.mmer(mixmodel, classify="germplasmName") ##runs the prediction + p0 = predict.mmer(mixmodel, D="germplasmName") ##runs the prediction summary(p0) adj = p0$pvals ##obtains the predictions adjustedBLUE = as.data.frame(adj) diff --git a/R/mixed_models_spl2D.R b/R/mixed_models_spl2D.R new file mode 100644 index 0000000000..026e342867 --- /dev/null +++ b/R/mixed_models_spl2D.R @@ -0,0 +1,162 @@ +## File was edited in ESS mode in emacs and some indents changed. sorry. +##First read in the arguments listed at the command line +args=commandArgs(TRUE) + +##args is now a list of character vectors +## First check to see if arguments are passed. +## Then cycle through each element of the list and evaluate the expressions. + +if(length(args)==0){ + print("No arguments supplied.") + ##supply default values + datafile='' + paramfile='' + fixed_factors<-c() + random_factors<-c() + +} else { + for(i in 1:length(args)){ + print(paste("Processing arg ", args[[i]])); + eval(parse(text=args[[i]])) + } +} + + +workdir = dirname(datafile); +setwd(workdir); + +#library(lme4) +#library(lmerTest) +library(sommer) +library(emmeans) +#library(effects) +library(stringr) +library(dplyr) + +pd = read.csv(datafile, sep="\t") + +source(paramfile) # should give us dependent_variable and the model + +trait = dependent_variables +pd$studyYear = as.factor(pd$studyYear) + +##Replacing location name to locationDbId +fixed_model <- gsub("locationName", "locationDbId", fixed_model) +random_model <- gsub("locationName", "locationDbId", random_model) +fixed_factors <- gsub("locationName", "locationDbId", fixed_factors) +random_factors <- gsub("locationName", "locationDbId", random_factors) + +print(paste("FIXED MODEL :", fixed_model)) +print(paste("RANDOM MODEL: ", random_model)) +print(paste("FIXED FACTORS: ", fixed_factors)) +print(paste("RANDOM FACTORS: ", random_factors)) +#print(head(pd)) + +BLUE = as.data.frame(unique(pd$germplasmName)) +colnames(BLUE) = "germplasmName" +adjustedBLUE = BLUE + +BLUPS = BLUE +adjusted_means = BLUE + +for(i in 1:length(trait)){ + + Rowf = as.factor(as.numeric(pd$rowNumber)) + Colf = as.factor(as.numeric(pd$colNumber)) + pd$rowNumber = as.numeric(pd$rowNumber) + pd$colNumber = as.numeric(pd$colNumber) + + #print(paste("processing trait", trait[i])) + dependent_variables = trait[i] + dependent_variables = gsub(" ", "\\.", dependent_variables) # replace space with "." in variable name + dependent_variables = gsub("\\|", "\\.", dependent_variables) # replace | with . + dependent_variables = gsub("\\:", "\\.", dependent_variables) + dependent_variables = gsub("\\-", "\\.", dependent_variables) + dependent_variables = gsub("\\/", "\\.", dependent_variables) + + print(paste("Dependent variables : ", dependent_variables)) + + pd <- pd[!(is.na(pd[c(dependent_variables)])), ] + + + + genotypeEffectType = as.vector(str_match(random_model, 'germplasmName')) + genotypeEffectType = ifelse(is.na(genotypeEffectType), 'fixed', 'random') + print(paste('modeling genotypes as: ', genotypeEffectType)) + + + if (genotypeEffectType=="random") { + + mixmodel = mmer(as.formula(fixed_model), random = as.formula(random_model), rcov = ~ vsr(units), dateWarning = F, data=pd) + + print(paste("MIXED MODEL: ", mixmodel)) + varcomp<- summary(mixmodel)$varcomp + print(varcomp) + + print("---------") + + print("---------") + + res <- mixmodel$U[["u:germplasmName"]] + BLUP <- as.data.frame(res) + BLUP <- tibble::rownames_to_column(BLUP, var="germplasmName") + # print(BLUP) + + ##ajusted means + adjusted_means = BLUP + adjusted_means[,2] = adjusted_means[,2] +as.numeric(mixmodel$Beta$Estimate[1]) + + print(adjusted_means) + + + } else { + + mixmodel = mmer(as.formula(fixed_model), random = as.formula(random_model), rcov = ~ vsr(units), dateWarning = F, data=pd) + + Blues <- summary(mixmodel)$beta + BLUE<-as.data.frame(Blues) + BLUE <- BLUE[,-1] + colnames(BLUE)[1] <- "accession name" + + Blues$Effect <- gsub("germplasmName","",Blues$Effect) + inter_name <- as.character(unique(pd$germplasmName[!unique(pd$germplasmName) %in% Blues$Effect])) + Blues$Effect[Blues$Effect == "(Intercept)"] <- inter_name[1] + + ##obtains adjusted blues + adjustedBLUE <- Blues[,c("Effect", "Estimate", "Std.Error")] + colnames(adjustedBLUE) <- c("accession name", "predicted value", "std.Error") + adjustedBLUE$`predicted value`[2:nrow(adjustedBLUE)] <- adjustedBLUE$`predicted value`[2:nrow(adjustedBLUE)] + mixmodel$Beta$Estimate[1] + + + BLUE$`accession name` <- adjustedBLUE$`accession name` + + print(adjustedBLUE) + + } +} + +# save result files +# +# for random effects: file.BLUPs and file.adjustedBLUPs +# for fixed effects: file.BLUEs and file.adjustedBLUEs +# +if (genotypeEffectType=="random") { + outfile_blup = paste(datafile, ".BLUPs", sep=""); + sink(outfile_blup) + write.table(BLUP, quote=F, sep='\t', row.names=FALSE) + sink(); + outfile_adjmeans = paste(datafile, ".adjustedBLUPs", sep="") + sink(outfile_adjmeans) + write.table(adjusted_means, quote=F , sep='\t', row.names=FALSE) + sink(); + +} else { #fixed + outfile_blue = paste(datafile, ".BLUEs", sep="") + sink(outfile_blue) + write.table(BLUE, quote=F , sep='\t', row.names=FALSE) + sink(); + outfile_adjblues = paste(datafile, ".adjustedBLUEs", sep="") + sink(outfile_adjblues) + write.table(adjustedBLUE, quote=F , sep='\t', row.names=FALSE) + sink(); +} diff --git a/R/solGS/anova.r b/R/solGS/anova.r index 3a89c028b7..8be7ab1663 100644 --- a/R/solGS/anova.r +++ b/R/solGS/anova.r @@ -24,13 +24,10 @@ inputFiles <- scan(grep("input_files", allArgs, value = TRUE), what = "character") phenoDataFile <- grep("phenotype_data", inputFiles, value = TRUE) -message('pheno file: ', phenoDataFile) traitsFile <- grep("traits", inputFiles, value = TRUE) -message('traits file: ', traitsFile) metadataFile <- grep("metadata", inputFiles, value = TRUE) -message('metadata file: ', metadataFile) metaData <- scan(metadataFile, what="character") @@ -53,33 +50,26 @@ traits <- strsplit(traits, "\t") for (trait in traits) { message('trait: ', trait) - anovaFiles <- grep("anova_table", + anovaTableFiles <- grep("anova_table", outputFiles, value = TRUE) - message('anova file: ', anovaFiles) - anovaHtmlFile <- grep("html", - anovaFiles, + anovaHtmlFile <- grep("anova_table_html", + anovaTableFiles, value = TRUE) - message('anova html file: ', anovaHtmlFile) - anovaTxtFile <- grep("txt", - anovaFiles, + anovaTxtFile <- grep("anova_table_txt", + anovaTableFiles, value = TRUE) - message('anova txt file: ', anovaTxtFile) modelSummFile <- grep("anova_model", outputFiles, value = TRUE) - message('model file: ', modelSummFile) adjMeansFile <- grep("adj_means", outputFiles, value = TRUE) - message('means file: ', adjMeansFile) - - diagnosticsFile <- grep("anova_diagnostics", outputFiles, value = TRUE) @@ -89,7 +79,6 @@ for (trait in traits) { value = TRUE) anovaOut <- runAnova(phenoData, trait) - if (class(anovaOut) == 'character') { cat(anovaOut, file=errorFile) } else if (is.null(anovaOut)) { @@ -111,12 +100,12 @@ for (trait in traits) { qqnorm(resid(anovaOut)) dev.off() - anovaTable <- getAnovaTable(anovaOut, + anovaHtmlTable <- getAnovaTable(anovaOut, tableType="html", traitName=trait, out=anovaHtmlFile) - - anovaTable <- getAnovaTable(anovaOut, + + anovaTxtTable <- getAnovaTable(anovaOut, tableType="text", traitName=trait, out=anovaTxtFile) diff --git a/R/solGS/correlation.r b/R/solGS/correlation.r index b396860966..b015a7546a 100644 --- a/R/solGS/correlation.r +++ b/R/solGS/correlation.r @@ -143,6 +143,9 @@ if (is.null(correInputData)) { stop("Can't run correlation analysis. There is no input data.") } +correInputData <- correInputData %>% + select(where(~n_distinct(.) > 2)) + coefpvalues <- rcor.test(correInputData, method="pearson", use="pairwise" diff --git a/R/solGS/gs.r b/R/solGS/gs.r deleted file mode 100644 index 6275c0ff57..0000000000 --- a/R/solGS/gs.r +++ /dev/null @@ -1,919 +0,0 @@ -#SNOPSIS -#calculates genomic estimated breeding values (GEBVs) using rrBLUP, -#GBLUP method - -#AUTHOR -# Isaak Y Tecle (iyt2@cornell.edu) - -options(echo = FALSE) -# options(warn = -1) -suppressWarnings(suppressPackageStartupMessages({ -library(methods) -library(rrBLUP) -library(plyr) -library(stringr) -library(randomForest) -library(parallel) -library(genoDataFilter) -library(phenoAnalysis) -library(caret) -library(dplyr) -library(tibble) -library(rlang) -library(jsonlite) -library(data.table) - })) -library(genoDataFilter) -library(Matrix) - -allArgs <- commandArgs() - - -inputFiles <- scan(grep("input_files", allArgs, value = TRUE), - what = "character") - -outputFiles <- scan(grep("output_files", allArgs, value = TRUE), - what = "character") - -traitsFile <- grep("traits", inputFiles, value = TRUE) -modelInfoFile <- grep("model_info", inputFiles, value = TRUE) -message('model_info_file ', modelInfoFile) - -modelInfo <- read.table(modelInfoFile, - header=TRUE, sep ="\t", - as.is = c('Value')) - -modelInfo <- column_to_rownames(modelInfo, var="Name") -traitId <- modelInfo["trait_id", 1] -traitAbbr <- modelInfo["trait_abbr", 1] -modelId <- modelInfo["model_id", 1] -protocolId <- modelInfo["protocol_id", 1] -protocolPage <- modelInfo["protocol_url", 1] - -message('trait_id ', traitId) -message('trait_abbr ', traitAbbr) -message('protocol_id ', protocolId) -message('protocol detail page ', protocolPage) - -message('model_id ', modelId) - -datasetInfoFile <- grep("dataset_info", inputFiles, value = TRUE) -datasetInfo <- c() - -if (length(datasetInfoFile) != 0 ) { - datasetInfo <- scan(datasetInfoFile, what = "character") - datasetInfo <- paste(datasetInfo, collapse = " ") - } else { - datasetInfo <- c('single_population') - } - -#validationTrait <- paste("validation", trait, sep = "_") -validationFile <- grep('validation', outputFiles, value = TRUE) - -if (is.null(validationFile)) { - stop("Validation output file is missing.") -} - -#kinshipTrait <- paste("rrblup_training_gebvs", trait, sep = "_") -blupFile <- grep('rrblup_training_gebvs', outputFiles, value = TRUE) - -if (is.null(blupFile)) { - stop("GEBVs file is missing.") -} - -#markerTrait <- paste("marker_effects", trait, sep = "_") -markerFile <- grep('marker_effects', outputFiles, value = TRUE) - -#traitPhenoFile <- paste("trait_phenotype_data", traitId, sep = "_") -modelPhenoFile <- grep('model_phenodata', outputFiles, value = TRUE) -message('model input trait pheno file ', modelPhenoFile) -modelGenoFile <- grep('model_genodata', outputFiles, value = TRUE) -message('model input trait geno file ', modelGenoFile) -traitRawPhenoFile <- grep('trait_raw_phenodata', outputFiles, value = TRUE) -varianceComponentsFile <- grep("variance_components", outputFiles, value = TRUE) -analysisReportFile <- grep("_report_", outputFiles, value = TRUE) -genoFilteringLogFile <- grep("genotype_filtering_log", outputFiles, value = TRUE) - -filteredTrainingGenoFile <- grep("filtered_training_genotype_data", outputFiles, value = TRUE) -filteredSelGenoFile <- grep("filtered_selection_genotype_data", outputFiles, value = TRUE) -formattedPhenoFile <- grep("formatted_phenotype_data", inputFiles, value = TRUE) - -genoFile <- grep("genotype_data_", inputFiles, value = TRUE) - -if (is.null(genoFile)) { - stop("genotype data file is missing.") -} - -if (file.info(genoFile)$size == 0) { - stop(paste0("genotype data file ", genoFile, " is empty.")) -} - -readfilteredTrainingGenoData <- c() -filteredTrainingGenoData <- c() -genoFilterLog <- c() -formattedPhenoData <- c() -phenoData <- c() -genoData <- c() -maf <- 0.01 -markerFilter <- 0.6 -cloneFilter <- 0.8 - -logHeading <- paste0("Genomic Prediction Analysis Log for ", traitAbbr, ".\n") -logHeading <- append(logHeading, paste0("Date: ", format(Sys.time(), "%d %b %Y %H:%M"), "\n\n\n")) -logHeading <- format(logHeading, width=80, justify="c") -trainingLog <- paste0("\n\n#Preprocessing training population genotype data.\n\n") -trainingLog <- append(trainingLog, "The following data filtering will be applied to the genotype dataset:\n\n") -trainingLog <- append(trainingLog, paste0("Markers with less or equal to ", maf * 100, "% minor allele frequency (maf) will be removed.\n")) -trainingLog <- append(trainingLog, paste0("\nMarkers with greater or equal to ", markerFilter * 100, "% missing values will be removed.\n")) -trainingLog <- append(trainingLog, paste0("Clones with greater or equal to ", cloneFilter * 100, "% missing values will be removed.\n") ) - -if (length(filteredTrainingGenoFile) != 0 && file.info(filteredTrainingGenoFile)$size != 0) { - filteredTrainingGenoData <- fread(filteredTrainingGenoFile, - na.strings = c("NA", "", "--", "-"), - header = TRUE) - - genoData <- data.frame(filteredTrainingGenoData) - genoData <- column_to_rownames(genoData, 'V1') - readfilteredTrainingGenoData <- 1 -} - -if (is.null(filteredTrainingGenoData)) { - genoData <- fread(genoFile, - na.strings = c("NA", "", "--", "-"), - header = TRUE) - genoData <- unique(genoData, by='V1') - genoData <- data.frame(genoData) - genoData <- column_to_rownames(genoData, 'V1') - #genoDataFilter::filterGenoData - genoData <- convertToNumeric(genoData) - - trainingLog <- append(trainingLog, "#Running training population genotype data cleaning.\n\n") - genoFilterOut <- filterGenoData(genoData, maf=maf, markerFilter=markerFilter, indFilter=cloneFilter, logReturn=TRUE) - - genoData <- genoFilterOut$data - genoFilteringLog <- genoFilterOut$log - genoData <- roundAlleleDosage(genoData) - filteredTrainingGenoData <- genoData - -} else { - genoFilteringLog <- scan(genoFilteringLogFile, what = "character", sep="\n") - genoFilteringLog <- paste0(genoFilteringLog, collapse="\n") -} - -message("genofilteringlogfile: ", genoFilteringLogFile) -message(genoFilteringLog) -trainingLog <- append(trainingLog, genoFilteringLog) - -genoData <- genoData[order(row.names(genoData)), ] - -if (length(formattedPhenoFile) != 0 && file.info(formattedPhenoFile)$size != 0) { - formattedPhenoData <- data.frame(fread(formattedPhenoFile, - header = TRUE, - na.strings = c("NA", "", "--", "-", ".") - )) - -} else { - - if (datasetInfo == 'combined_populations') { - - phenoFile <- grep("model_phenodata", inputFiles, value = TRUE) - } else { - - phenoFile <- grep("\\/phenotype_data", inputFiles, value = TRUE) - } - - if (is.null(phenoFile)) { - stop("phenotype data file is missing.") - } - - if (file.info(phenoFile)$size == 0) { - stop(paste0("phenotype data file ", phenoFile, " is empty.")) - - } - - phenoData <- data.frame(fread(phenoFile, - sep = "\t", - na.strings = c("NA", "", "--", "-", "."), - header = TRUE)) - - -} - -phenoTrait <- c() -traitRawPhenoData <- c() -anovaLog <- paste0("#Preprocessing training population phenotype data.\n\n") - -if (datasetInfo == 'combined_populations') { - anovaLog <- scan(analysisReportFile, what = "character", sep="\n") - anovaLog <- paste0(anovaLog, collapse="\n") - - if (!is.null(formattedPhenoData)) { - phenoTrait <- subset(formattedPhenoData, select = traitAbbr) - phenoTrait <- na.omit(phenoTrait) - - } else { - - if (any(grepl('Average', names(phenoData)))) { - phenoTrait <- phenoData %>% select(V1, Average) %>% data.frame - } else { - phenoTrait <- phenoData - } - - colnames(phenoTrait) <- c('genotypes', traitAbbr) - } - } else { - - if (!is.null(formattedPhenoData)) { - phenoTrait <- subset(formattedPhenoData, select = c('V1', traitAbbr)) - phenoTrait <- as.data.frame(phenoTrait) - phenoTrait <- na.omit(phenoTrait) - colnames(phenoTrait)[1] <- 'genotypes' - - } else if (length(grep('list', phenoFile)) != 0) { - phenoTrait <- averageTrait(phenoData, traitAbbr) - - } else { - meansResult <- getAdjMeans(phenoData, - traitName = traitAbbr, - calcAverages = TRUE, - logReturn = TRUE) - - - - anovaLog <- paste0(anovaLog, meansResult$log) - phenoTrait <- meansResult$adjMeans - } - - keepMetaCols <- c('observationUnitName', 'germplasmName', 'studyDbId', 'locationName', - 'studyYear', 'replicate', 'blockNumber', traitAbbr) - - traitRawPhenoData <- phenoData %>% - select(all_of(keepMetaCols)) - - -} - -meanType <- names(phenoTrait)[2] -names(phenoTrait) <- c('genotypes', traitAbbr) - -selectionTempFile <- grep("selection_population", inputFiles, value = TRUE) - -selectionFile <- c() -filteredPredGenoFile <- c() -selectionAllFiles <- c() - -if (length(selectionTempFile) !=0 ) { - selectionAllFiles <- scan(selectionTempFile, what = "character") - - selectionFile <- grep("\\/genotype_data", selectionAllFiles, value = TRUE) - - #filteredPredGenoFile <- grep("filtered_genotype_data_", selectionAllFiles, value = TRUE) -} - -selectionPopGEBVsFile <- grep("rrblup_selection_gebvs", outputFiles, value = TRUE) - -selectionData <- c() -readFilteredPredGenoData <- c() -filteredPredGenoData <- c() - -## if (length(filteredPredGenoFile) != 0 && file.info(filteredPredGenoFile)$size != 0) { -## selectionData <- fread(filteredPredGenoFile, na.strings = c("NA", " ", "--", "-"),) -## readFilteredPredGenoData <- 1 - -## selectionData <- data.frame(selectionData) -## rownames(selectionData) <- selectionData[, 1] -## selectionData[, 1] <- NULL - -## } else -selectionLog <- c() -if (length(selectionFile) != 0) { -selectionLog <- append(selectionLog, paste0("#Data preprocessing of selection population genotype data.\n\n")) - - selectionData <- fread(selectionFile, - header = TRUE, - na.strings = c("NA", "", "--", "-")) - - selectionData <- data.frame(selectionData) - selectionData <- unique(selectionData, by='V1') - selectionData <- column_to_rownames(selectionData, 'V1') - selectionData <- convertToNumeric(selectionData) - - selectionLog <- append(selectionLog, paste0("Running selection population genotype data cleaning.")) - - selectionFilterOut <- filterGenoData(selectionData, maf=maf, markerFilter=markerFilter, indFilter=cloneFilter, logReturn=TRUE) - selectionData <- selectionFilterOut$data - selectionLog <- append(selectionLog, selectionFilterOut$log) - selectionData <- roundAlleleDosage(selectionData) -} - -#impute genotype values for obs with missing values, -genoDataMissing <- c() - -if (sum(is.na(genoData)) > 0) { - genoDataMissing<- c('yes') - - genoData <- na.roughfix(genoData) - genoData <- data.frame(genoData) -} - -#create phenotype and genotype datasets with -#common stocks only - -#extract observation lines with both -#phenotype and genotype data only. -trainingLog <- append(trainingLog, paste0("\n\n#Filtering for training population genotypes with both phenotype and marker data.\n\n")) -trainingLog <- append(trainingLog, paste0("After calculating trait averages, the training population phenotype dataset has ", length(rownames(phenoTrait)), " individuals.\n") ) -trainingLog <- append(trainingLog, paste0("After cleaning up for missing values, the training population genotype dataset has ", length(rownames(genoData)), " individuals.\n") ) - -commonObs <- intersect(phenoTrait$genotypes, row.names(genoData)) - -trainingLog <- append(trainingLog, paste0(length(commonObs), " individuals are shared in both phenotype and genotype datasets.\n")) - -#remove genotyped lines without phenotype data -genoDataFilteredObs <- genoData[(rownames(genoData) %in% commonObs), ] - -trainingLog <- append(trainingLog, paste0("After removing individuals without phenotype data, this genotype dataset has ", length(rownames(genoDataFilteredObs)), " individuals.\n")) - -#remove phenotyped lines without genotype data -phenoTrait <- phenoTrait[(phenoTrait$genotypes %in% commonObs), ] - -trainingLog <- append(trainingLog, paste0("After removing individuals without genotype data, this phenotype dataset has ", length(rownames(phenoTrait)), " individuals.\n" )) - -phenoTraitMarker <- data.frame(phenoTrait) -rownames(phenoTraitMarker) <- phenoTraitMarker[, 1] -phenoTraitMarker[, 1] <- NULL - -#impute missing data in prediction data - -selectionDataMissing <- c() -if (length(selectionData) != 0) { - #purge markers unique to both populations - trainingMarkers <- names(genoDataFilteredObs) - selectionMarkers <- names(selectionData) - -selectionLog <- append(selectionLog, paste0("#Comparing markers in the training and selection populations genotype datasets.\n\n" )) - -selectionLog <- append(selectionLog, paste0("The training population genotype dataset has ", length(trainingMarkers), " markers.\n" )) -selectionLog <- append(selectionLog, paste0("The selection population genotype dataset has ", length(selectionMarkers), " markers.\n" )) - -commonMarkers <- intersect(trainingMarkers, selectionMarkers) -selectionLog <- append(selectionLog, paste0("The training and selection populations genotype dataset have ", length(trainingMarkers), " markers in common.\n" )) - -genoDataFilteredObs <- subset(genoDataFilteredObs, select= commonMarkers) -selectionLog <- append(selectionLog, paste0("After filtering for shared markers, the training population genotype dataset has ", length(names(selectionData)), " markers.\n" )) - -selectionData <- subset(selectionData, select = commonMarkers) -selectionLog <- append(selectionLog, paste0("After filtering for shared markers, the selection population genotype dataset has ", length(names(selectionData)), " markers.\n" )) - - if (sum(is.na(selectionData)) > 0) { - selectionDataMissing <- c('yes') - selectionData <- na.roughfix(selectionData) - selectionData <- data.frame(selectionData) - } -} -#change genotype coding to [-1, 0, 1], to use the A.mat ) if [0, 1, 2] -genoTrCode <- grep("2", genoDataFilteredObs[1, ], value = TRUE) -if(length(genoTrCode)) { - genoData <- genoData - 1 - genoDataFilteredObs <- genoDataFilteredObs - 1 -} - -if (length(selectionData) != 0 ) { - genoSlCode <- grep("2", selectionData[1, ], value = TRUE) - if (length(genoSlCode) != 0 ) { - selectionData <- selectionData - 1 - } -} - -ordered.markerEffects <- c() -trGEBV <- c() -validationAll <- c() -combinedGebvsFile <- c() -allGebvs <- c() -modelPhenoData <- c() -relationshipMatrix <- c() - -#additive relationship model -#calculate the inner products for -#genotypes (realized relationship matrix) -relationshipMatrixFile <- grep("relationship_matrix_table", outputFiles, value = TRUE) -relationshipMatrixJsonFile <- grep("relationship_matrix_json", outputFiles, value = TRUE) - -traitRelationshipMatrixFile <- grep("relationship_matrix_adjusted_table", outputFiles, value = TRUE) -traitRelationshipMatrixJsonFile <- grep("relationship_matrix_adjusted_json", outputFiles, value = TRUE) - -inbreedingFile <- grep('inbreeding_coefficients', outputFiles, value=TRUE) -aveKinshipFile <- grep('average_kinship', outputFiles, value=TRUE) - -inbreeding <- c() -aveKinship <- c() - -if (length(relationshipMatrixFile) != 0) { - if (file.info(relationshipMatrixFile)$size > 0 ) { - relationshipMatrix <- data.frame(fread(relationshipMatrixFile, - header = TRUE)) - - rownames(relationshipMatrix) <- relationshipMatrix[, 1] - relationshipMatrix[, 1] <- NULL - colnames(relationshipMatrix) <- rownames(relationshipMatrix) - relationshipMatrix <- data.matrix(relationshipMatrix) - - } else { - relationshipMatrix <- A.mat(genoData) - diag(relationshipMatrix) <- diag(relationshipMatrix) %>% replace(., . < 1, 1) -relationshipMatrix <- relationshipMatrix %>% replace(., . <= 0, 0.00001) - - inbreeding <- diag(relationshipMatrix) - inbreeding <- inbreeding - 1 - inbreeding <- data.frame(inbreeding) - - inbreeding <- inbreeding %>% - rownames_to_column('genotypes') %>% - rename(Inbreeding = inbreeding) %>% - arrange(Inbreeding) %>% - mutate_at('Inbreeding', round, 3) %>% - column_to_rownames('genotypes') - } -} - -relationshipMatrix <- data.frame(relationshipMatrix) -colnames(relationshipMatrix) <- rownames(relationshipMatrix) - -relationshipMatrix <- rownames_to_column(relationshipMatrix, var="genotypes") -relationshipMatrix <- relationshipMatrix %>% mutate_if(is.numeric, round, 5) -relationshipMatrix <- column_to_rownames(relationshipMatrix, var="genotypes") - -traitRelationshipMatrix <- relationshipMatrix[(rownames(relationshipMatrix) %in% commonObs), ] -traitRelationshipMatrix <- traitRelationshipMatrix[, (colnames(traitRelationshipMatrix) %in% commonObs)] - -kinshipLog <- c() -if (any(eigen(traitRelationshipMatrix)$values < 0) ) { -kinshipLog <- paste0("\n\nNote: The kinship matrix of this dataset causes 'Not positive semi-definite error' while running the Cholesky decomposition. To fix this and run the modeling, a corrected positive semi-definite matrix was computed using the 'Matrix::nearPD' function. The negative eigen values from this decomposition nudged to positive values.\n\n") - -traitRelationshipMatrix <- Matrix::nearPD(as.matrix(traitRelationshipMatrix))$mat -} - -traitRelationshipMatrix <- data.matrix(traitRelationshipMatrix) - -nCores <- detectCores() - -if (nCores > 1) { - nCores <- (nCores %/% 2) -} else { - nCores <- 1 -} -varCompData <- c() -modelingLog <- paste0("\n\n#Training a model for ", traitAbbr, ".\n\n") -modelingLog <- append(modelingLog, paste0("The genomic prediction modeling follows a two-step approach. First trait average values, as described above, are computed for each genotype. This is followed by the model fitting on the basis of single phenotype value for each genotype entry and kinship matrix computed from their marker data.\n")) - -if (length(kinshipLog)) { -modelingLog <- append(modelingLog, paste0(kinshipLog)) -} - -if (length(selectionData) == 0) { - - trModel <- kin.blup(data = phenoTrait, - geno = 'genotypes', - pheno = traitAbbr, - K = traitRelationshipMatrix, - n.core = nCores, - PEV = TRUE - ) - -modelingLog <- paste0(modelingLog, "The model training is based on rrBLUP R package, version ", packageVersion('rrBLUP'), ". GEBVs are predicted using the kin.blup function and GBLUP method.\n\n") - - trGEBV <- trModel$g - trGEBVPEV <- trModel$PEV - trGEBVSE <- sqrt(trGEBVPEV) - trGEBVSE <- data.frame(round(trGEBVSE, 2)) - - trGEBV <- data.frame(round(trGEBV, 2)) - - colnames(trGEBVSE) <- c('SE') - colnames(trGEBV) <- traitAbbr - - trGEBVSE <- rownames_to_column(trGEBVSE, var="genotypes") - trGEBV <- rownames_to_column(trGEBV, var="genotypes") - - trGEBVSE <- full_join(trGEBV, trGEBVSE) - - trGEBVSE <- trGEBVSE %>% arrange(across(traitAbbr, desc)) - trGEBVSE <- column_to_rownames(trGEBVSE, var="genotypes") - - trGEBV <- trGEBV %>% arrange(across(traitAbbr, desc)) - trGEBV <- column_to_rownames(trGEBV, var="genotypes") - - phenoTraitMarker <- data.matrix(phenoTraitMarker) - genoDataFilteredObs <- data.matrix(genoDataFilteredObs) - - markerEffects <- mixed.solve(y = phenoTraitMarker, - Z = genoDataFilteredObs - ) - - ordered.markerEffects <- data.matrix(markerEffects$u) - ordered.markerEffects <- data.matrix(ordered.markerEffects [order (-ordered.markerEffects[, 1]), ]) - ordered.markerEffects <- round(ordered.markerEffects, 5) - - colnames(ordered.markerEffects) <- c("Marker Effects") - ordered.markerEffects <- data.frame(ordered.markerEffects) - - modelPhenoData <- data.frame(round(phenoTraitMarker, 2)) - - heritability <- round((trModel$Vg/(trModel$Ve + trModel$Vg)), 2) - additiveVar <- round(trModel$Vg, 2) - errorVar <- round(trModel$Ve, 2) - - varCompData <- c("\nAdditive genetic variance\t", additiveVar, "\n") - varCompData <- append(varCompData, c("Error variance\t", errorVar, "\n")) - varCompData <- append(varCompData, c("SNP heritability (h)\t", heritability, "\n")) - - combinedGebvsFile <- grep('selected_traits_gebv', outputFiles, ignore.case = TRUE,value = TRUE) - - if (length(combinedGebvsFile) != 0) { - fileSize <- file.info(combinedGebvsFile)$size - if (fileSize != 0 ) { - combinedGebvs <- data.frame(fread(combinedGebvsFile, - header = TRUE)) - - rownames(combinedGebvs) <- combinedGebvs[,1] - combinedGebvs[,1] <- NULL - - allGebvs <- merge(combinedGebvs, trGEBV, - by = 0, - all = TRUE - ) - - rownames(allGebvs) <- allGebvs[,1] - allGebvs[,1] <- NULL - } - } - -#cross-validation - - if (is.null(selectionFile)) { - genoNum <- nrow(phenoTrait) - - if (genoNum < 20 ) { - warning(genoNum, " is too small number of genotypes.") - } - - set.seed(4567) - - k <- 10 - reps <- 2 - cvFolds <- createMultiFolds(phenoTrait[, 2], k=k, times=reps) - - modelingLog <- paste0(modelingLog, "Model prediction accuracy is evaluated using cross-validation method. ", k, " folds, replicated ", reps, " times are used to predict the model accuracy.\n\n") - - for ( r in 1:reps) { - re <- paste0('Rep', r) - - for (i in 1:k) { - fo <- ifelse(i < 10, 'Fold0', 'Fold') - - trFoRe <- paste0(fo, i, '.', re) - trG <- cvFolds[[trFoRe]] - slG <- as.numeric(rownames(phenoTrait[-trG,])) - - kblup <- paste("rKblup", i, sep = ".") - - result <- kin.blup(data = phenoTrait[trG,], - geno = 'genotypes', - pheno = traitAbbr, - K = traitRelationshipMatrix, - n.core = nCores, - PEV = TRUE - ) - - assign(kblup, result) - - #calculate cross-validation accuracy - valBlups <- result$g - valBlups <- data.frame(valBlups) - - slG <- slG[which(slG <= nrow(phenoTrait))] - - slGDf <- phenoTrait[(rownames(phenoTrait) %in% slG),] - rownames(slGDf) <- slGDf[, 1] - slGDf[, 1] <- NULL - - valBlups <- rownames_to_column(valBlups, var="genotypes") - slGDf <- rownames_to_column(slGDf, var="genotypes") - - valCorData <- inner_join(slGDf, valBlups, by="genotypes") - valCorData$genotypes <- NULL - - accuracy <- try(cor(valCorData)) - validation <- paste("validation", trFoRe, sep = ".") - cvTest <- paste("CV", trFoRe, sep = " ") - - if (inherits(accuracy, 'try-error') == FALSE) - { - accuracy <- round(accuracy[1,2], digits = 3) - accuracy <- data.matrix(accuracy) - - colnames(accuracy) <- c("correlation") - rownames(accuracy) <- cvTest - - assign(validation, accuracy) - - if (!is.na(accuracy[1,1])) { - validationAll <- rbind(validationAll, accuracy) - } - } - } - } - - validationAll <- data.frame(validationAll[order(-validationAll[, 1]), ]) - colnames(validationAll) <- c('Correlation') - } -} - -selectionPopResult <- c() -selectionPopGEBVs <- c() -selectionPopGEBVSE <- c() - -#selection pop geno data after cleaning up and removing unique markers to selection pop -filteredSelGenoData <- selectionData -if (length(selectionData) != 0) { - - genoDataTrSl <- rbind(genoDataFilteredObs, selectionData) - rTrSl <- A.mat(genoDataTrSl) - - selectionPopResult <- kin.blup(data = phenoTrait, - geno = 'genotypes', - pheno = traitAbbr, - K = rTrSl, - n.core = nCores, - PEV = TRUE - ) - - selectionPopGEBVs <- round(data.frame(selectionPopResult$g), 2) - colnames(selectionPopGEBVs) <- traitAbbr - selectionPopGEBVs <- rownames_to_column(selectionPopGEBVs, var="genotypes") - - selectionPopPEV <- selectionPopResult$PEV - selectionPopSE <- sqrt(selectionPopPEV) - selectionPopSE <- data.frame(round(selectionPopSE, 2)) - colnames(selectionPopSE) <- 'SE' - genotypesSl <- rownames(selectionData) - - selectionPopSE <- rownames_to_column(selectionPopSE, var="genotypes") - selectionPopSE <- selectionPopSE %>% filter(genotypes %in% genotypesSl) - - selectionPopGEBVs <- selectionPopGEBVs %>% filter(genotypes %in% genotypesSl) - - selectionPopGEBVSE <- inner_join(selectionPopGEBVs, selectionPopSE, by="genotypes") - - sortVar <- parse_expr(traitAbbr) - selectionPopGEBVs <- selectionPopGEBVs %>% arrange(desc((!!sortVar))) - selectionPopGEBVs <- column_to_rownames(selectionPopGEBVs, var="genotypes") - - selectionPopGEBVSE <- selectionPopGEBVSE %>% arrange(desc((!!sortVar))) - selectionPopGEBVSE <- column_to_rownames(selectionPopGEBVSE, var="genotypes") -} - -if (!is.null(selectionPopGEBVs) & length(selectionPopGEBVsFile) != 0) { - fwrite(selectionPopGEBVs, - file = selectionPopGEBVsFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if(!is.null(validationAll)) { - fwrite(validationAll, - file = validationFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - - -if (!is.null(ordered.markerEffects)) { - fwrite(ordered.markerEffects, - file = markerFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - - -if (!is.null(trGEBV)) { - fwrite(trGEBV, - file = blupFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (length(combinedGebvsFile) != 0 ) { - if(file.info(combinedGebvsFile)$size == 0) { - fwrite(trGEBV, - file = combinedGebvsFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) - } else { - fwrite(allGebvs, - file = combinedGebvsFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) - } -} - -if (!is.null(modelPhenoData) && length(modelPhenoFile) != 0) { - - if (!is.null(meanType)) { - colnames(modelPhenoData) <- meanType - } - - fwrite(modelPhenoData, - file = modelPhenoFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (!is.null(genoDataFilteredObs) && length(modelGenoFile) != 0) { - - fwrite(genoDataFilteredObs, - file = modelGenoFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (!is.null(traitRawPhenoData) && length(traitRawPhenoFile) != 0) { - - fwrite(traitRawPhenoData, - file = traitRawPhenoFile, - row.names = FALSE, - sep = "\t", - na = 'NA', - quote = FALSE, - ) -} - -if (!is.null(filteredTrainingGenoData) && file.info(filteredTrainingGenoFile)$size == 0) { - fwrite(filteredTrainingGenoData, - file = filteredTrainingGenoFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) - - cat(genoFilteringLog, fill = TRUE, file = genoFilteringLogFile, append=FALSE) -} - -if (length(filteredSelGenoFile) != 0 && file.info(filteredSelGenoFile)$size == 0) { - fwrite(filteredSelGenoData, - file = filteredSelGenoFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -## if (!is.null(genoDataMissing)) { -## write.table(genoData, -## file = genoFile, -## sep = "\t", -## col.names = NA, -## quote = FALSE, -## ) - -## } - -## if (!is.null(predictionDataMissing)) { -## write.table(predictionData, -## file = predictionFile, -## sep = "\t", -## col.names = NA, -## quote = FALSE, -## ) -## } - -if (file.info(relationshipMatrixFile)$size == 0) { - - fwrite(relationshipMatrix, - file = relationshipMatrixFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (file.info(relationshipMatrixJsonFile)$size == 0) { - - relationshipMatrixJson <- relationshipMatrix - relationshipMatrixJson[upper.tri(relationshipMatrixJson)] <- NA - - - relationshipMatrixJson <- data.frame(relationshipMatrixJson) - - relationshipMatrixList <- list(labels = names(relationshipMatrixJson), - values = relationshipMatrixJson) - - relationshipMatrixJson <- jsonlite::toJSON(relationshipMatrixList) - - - write(relationshipMatrixJson, - file = relationshipMatrixJsonFile, - ) -} - -if (file.info(traitRelationshipMatrixFile)$size == 0) { - - inbre <- diag(traitRelationshipMatrix) - inbre <- inbre - 1 - - diag(traitRelationshipMatrix) <- inbre - - traitRelationshipMatrix <- data.frame(traitRelationshipMatrix) %>% replace(., . < 0, 0) - - fwrite(traitRelationshipMatrix, - file = traitRelationshipMatrixFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) - - if (file.info(traitRelationshipMatrixJsonFile)$size == 0) { - - traitRelationshipMatrixJson <- traitRelationshipMatrix - traitRelationshipMatrixJson[upper.tri(traitRelationshipMatrixJson)] <- NA - - traitRelationshipMatrixJson <- data.frame(traitRelationshipMatrixJson) - - traitRelationshipMatrixList <- list(labels = names(traitRelationshipMatrixJson), - values = traitRelationshipMatrixJson) - - traitRelationshipMatrixJson <- jsonlite::toJSON(traitRelationshipMatrixList) - - write(traitRelationshipMatrixJson, - file = traitRelationshipMatrixJsonFile, - ) - } -} - - -if (file.info(inbreedingFile)$size == 0) { - - fwrite(inbreeding, - file = inbreedingFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (file.info(aveKinshipFile)$size == 0) { - - aveKinship <- data.frame(apply(traitRelationshipMatrix, 1, mean)) - - aveKinship<- aveKinship %>% - rownames_to_column('genotypes') %>% - rename(Mean_kinship = contains('traitRe')) %>% - arrange(Mean_kinship) %>% - mutate_at('Mean_kinship', round, 3) %>% - column_to_rownames('genotypes') - - fwrite(aveKinship, - file = aveKinshipFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} - -if (file.info(formattedPhenoFile)$size == 0 && !is.null(formattedPhenoData) ) { - fwrite(formattedPhenoData, - file = formattedPhenoFile, - row.names = TRUE, - sep = "\t", - quote = FALSE, - ) -} -if (!is.null(varCompData)) { - cat(varCompData, file = varianceComponentsFile) -} - -if (!is.null(selectionLog)) { - cat(logHeading, selectionLog, fill = TRUE, file = analysisReportFile, append=FALSE) -} else { - cat(logHeading, anovaLog, trainingLog, modelingLog, fill = TRUE, file = analysisReportFile, append=FALSE) -} - -message("Done.") - -q(save = "no", runLast = FALSE) diff --git a/R/solGS/hclustering.r b/R/solGS/hclustering.r index 57073667e2..464f8eb836 100644 --- a/R/solGS/hclustering.r +++ b/R/solGS/hclustering.r @@ -39,7 +39,7 @@ errorFile <- grep("error", outputFiles, value = TRUE) combinedDataFile <- grep("combined_cluster_data_file", outputFiles, value = TRUE) -plotFile <- grep("plot", outputFiles, value = TRUE) +plotFile <- grep("hierarchical_plot", outputFiles, value = TRUE) optionsFile <- grep("options", inputFiles, value = TRUE) clusterOptions <- read.table(optionsFile, @@ -99,7 +99,6 @@ extractGenotype <- function(inputFiles) { stop("There is no genotype dataset.") q("no", 1, FALSE) } else { - ##genoDataFilter::filterGenoData genoData <- convertToNumeric(genoData) genoData <- filterGenoData(genoData, maf=0.01) @@ -194,32 +193,27 @@ if (length(sIndexFile) != 0) { } distMat <- clusterData %>% - dist(., method="euclidean") + dist(., method="euclidean") distMat <- round(distMat, 3) - hClust <- distMat %>% - hclust(., method="complete") - + hclust(., method="complete") distTable <- data.frame(as.matrix(distMat)) -clustTree <- ggtree(hClust, layout = "circular", color = "#96CA2D") +clustTree <- ggtree::ggtree(hClust, layout = "circular", color = "#96CA2D") xMax <- ggplot_build(clustTree)$layout$panel_scales_x[[1]]$range$range[2] xMax <- xMax + 0.02 # ggplot2::xlim(0, xMax) clustTree <- clustTree + - geom_tiplab(size = 4, color = "blue") - + geom_tiplab(size = 3, color = "blue") # geom_text(aes(x = branch, label = round(branch.length, 2))) - -png(plotFile, height = 950, width = 950) - print(clustTree) +png(filename=plotFile, height = 950, width = 950) +print(clustTree) dev.off() - cat(reportNotes, file = reportFile, sep = "\n", append = TRUE) if (length(genoFiles) > 1) { @@ -238,15 +232,16 @@ if (length(genoFiles) > 1) { # quote = FALSE, # ) # } -message('newick file ', newickFile) -newickF <- as.phylo(hClust) -write.tree(phy = newickF, + +newickFormat <- ape::as.phylo(hClust) +write.tree(phy = newickFormat, file = newickFile ) -jsonHclust <- jsonHC(hClust) +jsonHclust <- R2D3::jsonHC(hClust) write(jsonHclust$json, file = jsonFile) +message("Done hierachical clustering.") #### q(save = "no", runLast = FALSE) #### diff --git a/R/solGS/kclustering.r b/R/solGS/kclustering.r index 7dabb3a22f..8162342dc0 100644 --- a/R/solGS/kclustering.r +++ b/R/solGS/kclustering.r @@ -37,6 +37,7 @@ kmeansPlotFile <- grep("k-means_plot", outputFiles, value = TRUE) kResultFile <- grep("result", outputFiles, value = TRUE) elbowPlotFile <- grep("elbow_plot", outputFiles, value = TRUE) clusterMeansFile <- grep("k-means_means", outputFiles, value = TRUE) +clusterPcScoresFile <- grep("k-means_pc_scores", outputFiles, value = TRUE) variancesFile <- grep("k-means_variances", outputFiles, value = TRUE) reportFile <- grep("report", outputFiles, value = TRUE) errorFile <- grep("error", outputFiles, value = TRUE) @@ -76,7 +77,7 @@ extractGenotype <- function(inputFiles) { } else { genoFile <- genoFiles - genoData <- fread(genoFile, header = TRUE, na.strings = c("NA", " ", "--", + genoData <- fread(genoFile, header = TRUE, na.strings = c("NA", "", "--", "-", ".")) if (is.null(genoData)) { @@ -93,11 +94,9 @@ extractGenotype <- function(inputFiles) { stop("There is no genotype dataset.") q("no", 1, FALSE) } else { - - ## genoDataFilter::filterGenoData - genoData <- convertToNumeric(genoData) - genoData <- filterGenoData(genoData, maf = 0.01) - genoData <- roundAlleleDosage(genoData) + genoData <- genoDataFilter::convertToNumeric(genoData) + genoData <- genoDataFilter::filterGenoData(genoData, maf = 0.01) + genoData <- genoDataFilter::roundAlleleDosage(genoData) message("No. of geno missing values, ", sum(is.na(genoData))) if (sum(is.na(genoData)) > 0) { @@ -115,36 +114,7 @@ clusterDataNotScaled <- c() if (grepl("genotype", dataType, ignore.case = TRUE)) { clusterData <- extractGenotype(inputFiles) - - pca <- prcomp(clusterData, retx = TRUE) - pca <- summary(pca) - - variances <- data.frame(pca$importance) - - varProp <- variances[3, ] - varProp <- data.frame(t(varProp)) - names(varProp) <- c("cumVar") - - selectPcs <- varProp %>% - filter(cumVar <= 0.9) - pcsCnt <- nrow(selectPcs) - - reportNotes <- paste0("Before clustering this dataset, principal component analysis (PCA) was perforemd on it to reduce the number of variables (dimensions). ") - reportNotes <- paste0(reportNotes, "Based on the PCA, ", pcsCnt, " PCs were used to do the clustering. ") - reportNotes <- paste0(reportNotes, "\n\nThe ", pcsCnt, " PCs explain 90% of the variance in the original dataset.", - "\n") - - scores <- data.frame(pca$x) - scores <- scores[, 1:pcsCnt] - scores <- round(scores, 3) - - variances <- variances[2, 1:pcsCnt] - variances <- round(variances, 4) * 100 - variances <- data.frame(t(variances)) - - clusterData <- scores } else { - if (grepl("gebv", dataType, ignore.case = TRUE)) { gebvsFile <- grep("combined_gebvs", inputFiles, value = TRUE) gebvsData <- data.frame(fread(gebvsFile, header = TRUE)) @@ -156,9 +126,9 @@ if (grepl("genotype", dataType, ignore.case = TRUE)) { metaFile <- grep("meta", inputFiles, value = TRUE) - clusterData <- cleanAveragePhenotypes(inputFiles, metaDataFile = metaFile) + clusterData <- phenoAnalysis::cleanAveragePhenotypes(inputFiles, metaDataFile = metaFile) - if (!is.na(predictedTraits) && length(predictedTraits) > 1) { + if (length(predictedTraits) > 1) { clusterData <- rownames_to_column(clusterData, var = "germplasmName") clusterData <- clusterData %>% select(c(germplasmName, predictedTraits)) @@ -193,21 +163,38 @@ if (length(sIndexFile) != 0) { } } -kMeansOut <- kmeansruns(clusterData, runs = 10) +kMeansOut <-fpc::kmeansruns(clusterData, runs = 10) kCenters <- kMeansOut$bestk - -if (!is.na(userKNumbers)) { +recommendedK <- kMeansOut$bestk + +message("recommended number of clusters: ", kCenters) +message("user defined number of clusters: ", userKNumbers) +if (is.na(userKNumbers)) { + reportNotes <- paste0(reportNotes, "\n\nYou provided no cluster numbers (k): ", userKNumbers, "\n") + reportNotes <- paste0(reportNotes, "\n\nThe data was partitioned into ", recommendedK, + " clusters, the recommended number of clusters (k) for the given data ", + "based on the the kmeansruns algorithm (with runs=10) ", + "from the fpc R package.\n") +} else { if (userKNumbers != 0) { kCenters <- as.integer(userKNumbers) - reportNotes <- paste0(reportNotes, "\n\nThe data was partitioned into ", userKNumbers, - " clusters.\n") + reportNotes <- paste0(reportNotes, "\n\nThe data was partitioned into ", + kCenters, " clusters, the number of clusters ", + "(k) defined by you.\n") + } else { + stop("The number of clusters (k) should be a whole number. Provided k: ", + userKNumbers, "\n") + q("no", 1, FALSE) } } -reportNotes <- paste0(reportNotes, "\n\nAccording the kmeansruns algorithm from the fpc R package, the recommended number of clusters (k) for this data set is: ", - kCenters, "\n\nYou can also check the Elbow plot to evaluate how many clusters may be better suited for your purpose.") +reportNotes <- paste0(reportNotes, "\n\nAccording the kmeansruns algorithm ", +"(with runs=10) from the fpc R package, the recommended ", +"number of clusters (k) for this data set is: ", +recommendedK, "\n\nYou can also check the Elbow plot to evaluate ", +"how many clusters may be better suited for your purpose.") -kMeansOut <- kmeans(clusterData, centers = kCenters, nstart = 10) +kMeansOut <- stats::kmeans(clusterData, centers = kCenters, nstart = 10) kClusters <- data.frame(kMeansOut$cluster) kClusters <- rownames_to_column(kClusters) names(kClusters) <- c("germplasmName", "Cluster") @@ -233,7 +220,7 @@ clusteredData <- clusteredData %>% if (length(elbowPlotFile) && !file.info(elbowPlotFile)$size) { message("running elbow method...") png(elbowPlotFile) - print(fviz_nbclust(clusterData, k.max = 20, FUNcluster = kmeans, method = "wss")) + print(factoextra::fviz_nbclust(clusterData, k.max = 20, FUNcluster = kmeans, method = "wss")) dev.off() } @@ -244,7 +231,7 @@ dev.off() clusterMeans <- c() if (!grepl('genotype', kResultFile)) { - message('adding cluster means to clusters...') + message("adding cluster means to clusters...") clusterMeans <- aggregate(clusterDataNotScaled, by = list(cluster = kMeansOut$cluster), mean) @@ -254,6 +241,31 @@ if (!grepl('genotype', kResultFile)) { } +pca <- c() +if (grepl("genotype", dataType, ignore.case = TRUE)) { + pca <- prcomp(clusterData, retx=TRUE) +} else if (is.null(selectedIndexGenotypes)) { + pca <- prcomp(clusterData, scale=TRUE, retx=TRUE) +} else { + pca <- prcomp(clusterData, retx=TRUE) +} + +pca <- summary(pca) +scores <- data.frame(pca$x) +scores <- scores[, 1:2] +scores <- round(scores, 3) + +clusterPcScoresGroups <- c() +if (length(clusterPcScoresFile)) { + message("adding cluster groups to pc scores...") + scores <- rownames_to_column(scores) + names(scores)[1] <- c("germplasmName") + + clusterPcScoresGroups <- inner_join(kClusters, scores, by = "germplasmName") + clusterPcScoresGroups <- clusterPcScoresGroups %>% + arrange(Cluster) +} + cat(reportNotes, file = reportFile, sep = "\n", append = TRUE) variances <- paste0("Variances output: ") @@ -286,6 +298,11 @@ if (length(clusterMeansFile) && !is.null(clusterMeans)) { quote = FALSE, ) } +if (length(clusterPcScoresFile) && !is.null(clusterPcScoresGroups)) { + fwrite(clusterPcScoresGroups, file = clusterPcScoresFile, sep = "\t", row.names = FALSE, + quote = FALSE, ) +} + message("Done clustering.") diff --git a/R/solGS/kinship.r b/R/solGS/kinship.r index 75d2d2a1b9..227257f4fd 100644 --- a/R/solGS/kinship.r +++ b/R/solGS/kinship.r @@ -41,7 +41,7 @@ createGenoData <- function(inputFiles) { filteredGenoFile <- c() if (length(genoFiles) > 1) { - genoData <- combineGenoData(genoFiles) + genoData <- genoDataFilter::combineGenoData(genoFiles) genoMetaData <- genoData$trial genoData$trial <- NULL @@ -70,9 +70,9 @@ createGenoData <- function(inputFiles) { } else { ##genoDataFilter::filterGenoData - genoData <- convertToNumeric(genoData) - genoData <- filterGenoData(genoData, maf=0.01) - genoData <- roundAlleleDosage(genoData) + genoData <- genoDataFilter::convertToNumeric(genoData) + genoData <- genoDataFilter::filterGenoData(genoData, maf=0.01) + genoData <- genoDataFilter::roundAlleleDosage(genoData) message("No. of geno missing values, ", sum(is.na(genoData))) if (sum(is.na(genoData)) > 0) { @@ -110,7 +110,7 @@ aveKinship <- c() relationshipMatrixJson <- c() -relationshipMatrix <- A.mat(genoData) +relationshipMatrix <- rrBLUP::A.mat(genoData) diag(relationshipMatrix) <- diag(relationshipMatrix) + 1e-6 genos <- rownames(relationshipMatrix) @@ -156,9 +156,6 @@ aveKinship <- aveKinship %>% relationshipMatrixJson <- relationshipMatrix relationshipMatrixJson[upper.tri(relationshipMatrixJson)] <- NA - - -#relationshipMatrixJson <- data.frame(relationshipMatrixJson) relationshipMatrixList <- list(labels = names(relationshipMatrixJson), values = relationshipMatrixJson) relationshipMatrixJson <- jsonlite::toJSON(relationshipMatrixList) diff --git a/R/solGS/pca.r b/R/solGS/pca.r index 5320e747b6..16d1409a1e 100644 --- a/R/solGS/pca.r +++ b/R/solGS/pca.r @@ -26,20 +26,24 @@ inputFile <- grep("input_files", allArgs, value = TRUE) inputFiles <- scan(inputFile, what = "character") scoresFile <- grep("pca_scores", outputFiles, value = TRUE) -screeDataFile <- grep("pca_scree_data", outputFiles, value = TRUE) -screeFile <- grep("pca_scree_plot", outputFiles, value = TRUE) +screeDataFile <- grep("pca_scree_data", outputFiles, value = TRUE) +screeFile <- grep("pca_scree_plot", outputFiles, value = TRUE) loadingsFile <- grep("pca_loadings", outputFiles, value = TRUE) varianceFile <- grep("pca_variance", outputFiles, value = TRUE) combinedDataFile <- grep("combined_pca_data_file", outputFiles, value = TRUE) +reportFile <- grep("report", outputFiles, value = TRUE) -if (is.null(scoresFile)) -{ +if (is.null(inputFiles)) { + stop("Input files are missing.") + q("no", 1, FALSE) +} + +if (is.null(scoresFile)) { stop("Scores output file is missing.") q("no", 1, FALSE) } -if (is.null(loadingsFile)) -{ +if (is.null(loadingsFile)) { stop("Laodings file is missing.") q("no", 1, FALSE) } @@ -48,17 +52,16 @@ genoData <- c() genoMetaData <- c() filteredGenoFile <- c() phenoData <- c() +phenoDataNotes <- c() set.seed(235) -pcF <- grepl("genotype", ignore.case=TRUE, inputFiles) -dataType <- ifelse(isTRUE(pcF[1]), 'genotype', 'phenotype') +pcaDataFile <- grepl("genotype", ignore.case=TRUE, inputFiles) +dataType <- ifelse(isTRUE(pcaDataFile[1]), 'genotype', 'phenotype') if (dataType == 'genotype') { - if (length(inputFiles) > 1 ) { - allGenoFiles <- inputFiles - genoData <- combineGenoData(allGenoFiles) - + if (length(inputFiles) > 1) { + genoData <- genoDataFilter::combineGenoData(inputFiles) genoMetaData <- genoData$trial genoData$trial <- NULL @@ -68,9 +71,7 @@ if (dataType == 'genotype') { header = TRUE, na.strings = c("NA", " ", "--", "-", ".")) - if (is.null(genoData)) { - filteredGenoFile <- grep("filtered_genotype_data_", genoDataFile, value = TRUE) @@ -87,13 +88,13 @@ if (dataType == 'genotype') { } } else if (dataType == 'phenotype') { - metaFile <- grep("meta", inputFiles, value = TRUE) + metaDataFile <- grep("meta", inputFiles, value = TRUE) phenoFiles <- grep("phenotype_data", inputFiles, value = TRUE) if (length(phenoFiles) > 1 ) { - phenoData <- combinePhenoData(phenoFiles, metaDataFile = metaFile) - phenoData <- summarizeTraits(phenoData, groupBy=c('studyDbId', 'germplasmName')) + phenoData <- phenoAnalysis::combinePhenoData(phenoFiles, metaDataFile = metaDataFile) + phenoData <- phenoAnalysis::summarizeTraits(phenoData, groupBy=c('studyDbId', 'germplasmName')) if (all(is.na(phenoData$locationName))) { phenoData$locationName <- 'location' @@ -102,7 +103,7 @@ if (dataType == 'genotype') { phenoData <- na.omit(phenoData) genoMetaData <- phenoData$studyDbId - phenoData <- phenoData %>% mutate(germplasmName = paste0(germplasmName, '_', studyDbId)) + phenoData <- phenoData %>% mutate(germplasmName = paste0(germplasmName, '_trial_', studyDbId)) dropCols = c('replicate', 'blockNumber', 'locationName', 'studyDbId', 'studyYear') phenoData <- phenoData %>% select(-dropCols) rownames(phenoData) <- NULL @@ -110,11 +111,23 @@ if (dataType == 'genotype') { } else { phenoDataFile <- grep("phenotype_data", inputFiles, value = TRUE) - phenoData <- cleanAveragePhenotypes(inputFiles, metaFile) + phenoData <- phenoAnalysis::cleanAveragePhenotypes(inputFiles, metaDataFile) phenoData <- na.omit(phenoData) } + traits <- colnames(phenoData) + phenoDataNotes <- paste0("\nThis PCA is done on phenotype data. \n") + phenoDataNotes <- append(phenoDataNotes, + paste0("\n\nTraits before removing zero variance traits: ", + paste(traits, collapse = ", "))) + phenoData <- phenoData[, apply(phenoData, 2, var) != 0 ] + traits <- colnames(phenoData) + phenoDataNotes <- append(phenoDataNotes, + paste0("\n\nTraits After removing zero variance traits: ", + paste(traits, collapse = ", "))) + phenoDataNotes <- append(phenoDataNotes, + paste0("\nTrait data is centered and scaled before PCA.\n")) phenoData <- scale(phenoData, center=TRUE, scale=TRUE) phenoData <- round(phenoData, 3) } @@ -125,31 +138,30 @@ if (is.null(genoData) && is.null(phenoData)) { q("no", 1, FALSE) } +notesHeader <- paste0("PCA Analysis Report") +notesHeader <- append(notesHeader, paste0("\nDate: ", + format(Sys.time(), "%d %b %Y %H:%M"), + "\n\n\n")) + +# notesHeader<- format(notesHeader, width = 80, justify = "c") + +genoProcessingNotes <- c() genoDataMissing <- c() if (dataType == 'genotype') { - # if (is.null(filteredGenoFile) == TRUE) { - ##genoDataFilter::filterGenoData - genoData <- convertToNumeric(genoData) - genoData <- filterGenoData(genoData, maf=0.01) - genoData <- roundAlleleDosage(genoData) - - message("No. of geno missing values, ", sum(is.na(genoData)) ) - if (sum(is.na(genoData)) > 0) { - genoDataMissing <- c('yes') - # genoData <- na.roughfix(genoData) - } + genoData <- genoDataFilter::convertToNumeric(genoData) + filterGenoDataResult <- genoDataFilter::filterGenoData(genoData, maf=0.01, logReturn = TRUE) + genoProcessingNotes <- paste0("\nPreprocessing genotype data before PCA\n") + genoProcessingNotes <- append(genoProcessingNotes, filterGenoDataResult$log) + genoData <- filterGenoDataResult$data + genoData <- genoDataFilter::roundAlleleDosage(genoData) + + if (sum(is.na(genoData)) > 0) { + genoDataMissing <- c('yes') + # genoData <- na.roughfix(genoData) + } } - -## nCores <- detectCores() -## message('no cores: ', nCores) -## if (nCores > 1) { -## nCores <- (nCores %/% 2) -## } else { -## nCores <- 1 -## } - pcaData <- c() if (!is.null(genoData)) { pcaData <- genoData @@ -159,13 +171,13 @@ if (!is.null(genoData)) { phenoData <- NULL } - message("No. of missing values, ", sum(is.na(pcaData)) ) if (sum(is.na(pcaData)) > 0) { pcaData <- na.roughfix(pcaData) } pcsCnt <- ifelse(ncol(pcaData) < 10, ncol(pcaData), 10) +pcaNotes <- paste0("\n\nPCA is done using the prcomp function in R.\n") pca <- prcomp(pcaData, retx=TRUE) pca <- summary(pca) @@ -175,7 +187,8 @@ scores <- round(scores, 3) if (!is.null(genoMetaData)) { scores$trial <- genoMetaData - scores <- scores %>% select(trial, everything()) %>% data.frame + scores <- scores %>% data.frame + scores <- scores %>% select(trial, everything()) %>% data.frame } else { scores$trial <- 1000 scores <- scores %>% select(trial, everything()) %>% data.frame @@ -237,16 +250,13 @@ if (!is.null(genoData)) { } } - -# ## if (!is.null(genoDataMissing)) { -# ## fwrite(genoData, -# ## file = genoDataFile, -# ## sep = "\t", -# ## row.names = TRUE, -# ## quote = FALSE, -# ## ) -# -# ## } - +cat(notesHeader, + genoProcessingNotes, + phenoDataNotes, + pcaNotes, + fill=TRUE, + file = reportFile, + append = FALSE +) q(save = "no", runLast = FALSE) diff --git a/R/solGS/rrblup_gblup_gs.r b/R/solGS/rrblup_gblup_gs.r new file mode 100644 index 0000000000..53f0828877 --- /dev/null +++ b/R/solGS/rrblup_gblup_gs.r @@ -0,0 +1,1465 @@ +#SNOPSIS +#calculates genomic estimated breeding values (GEBVs) using rrBLUP, +#GBLUP method + +#AUTHOR +# Isaak Y Tecle (iyt2@cornell.edu) + +options(echo = FALSE) +# options(warn = -1) +suppressWarnings(suppressPackageStartupMessages({ + library(methods) + library(rrBLUP) + library(plyr) + library(stringr) + library(randomForest) + library(parallel) + library(genoDataFilter) + library(phenoAnalysis) + library(caret) + library(dplyr) + library(tibble) + library(rlang) + library(jsonlite) + library(data.table) +})) + +library(genoDataFilter) +library(Matrix) + +all_args <- commandArgs() +input_files <- tryCatch({ + scan(grep("input_files", all_args, value = TRUE), what = "character") +}, error = function(e) { + stop("Input files are missing or do not exist.") +}) + +output_files <- tryCatch({ + scan(grep("output_files", all_args, value = TRUE), what = "character") +}, error = function(e) { + stop("Output files are missing or do not exist.") +}) + +# output_files <- scan(grep("output_files", allArgs, value = TRUE), +# what = "character") + +traits_file <- grep("traits", input_files, value = TRUE) + +model_info_file <- grep("model_info", input_files, value = TRUE) +message("model_info_file: ", model_info_file) + +model_info <- read.table( + model_info_file, + header = TRUE, + sep = "\t", + as.is = c("Value") +) + +model_info <- column_to_rownames(model_info, var="Name") +trait_id <- model_info["trait_id", 1] +trait_abbr <- model_info["trait_abbr", 1] +model_id <- model_info["model_id", 1] +protocol_id <- model_info["protocol_id", 1] +protocol_page <- model_info["protocol_url", 1] + +message("trait_id: ", trait_id) +message("trait_abbr: ", trait_abbr) +message("protocol_id: ", protocol_id) +message("protocol detail page: ", protocol_page) + +message("model_id: ", model_id) + +dataset_info_file <- grep("dataset_info", input_files, value = TRUE) +dataset_info <- c() + +if (length(dataset_info_file) != 0) { + dataset_info <- scan(dataset_info_file, what = "character") + dataset_info <- paste(dataset_info, collapse = " ") +} else { + dataset_info <- c("single_population") +} + +#validationTrait <- paste("validation", trait, sep = "_") +validation_file <- grep("validation", output_files, value = TRUE) + +if (is.null(validation_file)) { + stop("Validation output file is missing.") +} + +#kinshipTrait <- paste("rrblup_training_gebvs", trait, sep = "_") +blup_file <- grep("rrblup_training_gebvs", output_files, value = TRUE) + +if (length(blup_file) == 0) { + stop("GEBVs file is missing.") +} + +#markerTrait <- paste("marker_effects", trait, sep = "_") +marker_file <- grep("marker_effects", output_files, value = TRUE) + +#traitPhenoFile <- paste("trait_phenotype_data", trait_id, sep = "_") +model_pheno_file <- grep("model_phenodata", output_files, value = TRUE) +message("model input trait pheno file: ", model_pheno_file) +model_geno_file <- grep("model_genodata", output_files, value = TRUE) +message("model input trait geno file: ", model_geno_file) + +training_pop_genetic_values_file <- grep( + "training_genetic_values", + output_files, + value = TRUE +) + +combined_training_gebvs_genetic_values_file <- grep( + "combined_training_gebvs_genetic_values", + output_files, + value = TRUE +) + +selection_pop_genetic_values_file <- grep( + "selection_genetic_values", + output_files, + value = TRUE +) + +combined_selection_gebvs_genetic_values_file <- grep( + "combined_selection_gebvs_genetic_values", + output_files, + value = TRUE +) + +trait_raw_pheno_file <- grep( + "trait_raw_phenodata", + output_files, + value = TRUE +) + +variance_components_file <- grep( + "variance_components", + output_files, + value = TRUE +) +analysis_report_file <- grep("_report_", + output_files, + value = TRUE) +geno_filtering_log_file <- grep("genotype_filtering_log", + output_files, + value = TRUE) + +filtered_training_geno_file <- grep("filtered_training_genotype_data", + output_files, + value = TRUE) + +filtered_sel_geno_file <- grep("filtered_selection_genotype_data", + output_files, + value = TRUE) + +formatted_pheno_file <- grep("formatted_phenotype_data", + input_files, + value = TRUE) + +geno_file <- grep("genotype_data_", + input_files, + value = TRUE) + +if (is.null(geno_file)) { + stop("genotype data file is missing.") +} + +if (file.info(geno_file)$size == 0) { + stop(paste0("genotype data file ", geno_file, " is empty.")) +} + +read_filtered_training_geno_data <- c() +filtered_training_geno_data <- c() +geno_filter_log <- c() +formatted_pheno_data <- c() + +pheno_data <- c() +geno_data <- c() + +maf <- 0.01 +marker_filter_threshold <- 0.6 +pheno_filter_threshold <- 0.8 + +log_heading <- paste0("Genomic Prediction Analysis Log for ", + trait_abbr, + ".\n" +) + +message("log heading: ", log_heading) +log_heading <- append(log_heading, + paste0("Date: ", + format(Sys.time(), "%d %b %Y %H:%M"), + "\n\n\n") +) +message("log heading: ", log_heading) + +log_heading <- format(log_heading, width = 80, justify = "c") + +training_log <- paste0( + "\n\n#Preprocessing training population genotype data.\n\n" +) +training_log <- append( + training_log, + "The following data filtering will be applied to the genotype dataset:\n\n" +) +training_log <- append( + training_log, + paste0("Markers with less or equal to ", + maf * 100, + "% minor allele frequency (maf) will be removed.\n" + ) +) + +training_log <- append( + training_log, + paste0("\nMarkers with greater or equal to ", + marker_filter_threshold * 100, + "% missing values will be removed.\n" + ) +) +training_log <- append( + training_log, + paste0("Clones with greater or equal to ", + pheno_filter_threshold * 100, + "% missing values will be removed.\n" + ) +) + +if (length(filtered_training_geno_file) != 0 && + file.info(filtered_training_geno_file)$size != 0) { + filtered_training_geno_data <- fread(filtered_training_geno_file, + na.strings = c("NA", "", "--", "-"), + header = TRUE) + + geno_data <- data.frame(filtered_training_geno_data) + geno_data <- column_to_rownames(geno_data, "V1") + read_filtered_training_geno_data <- 1 +} + +if (is.null(filtered_training_geno_data)) { + geno_data <- fread(geno_file, + na.strings = c("NA", "", "--", "-"), + header = TRUE) + + + geno_data <- unique(geno_data, by = "V1") + geno_data <- data.frame(geno_data) + + geno_data <- column_to_rownames(geno_data, "V1") + #genoDataFilter::filterGenoData + geno_data <- genoDataFilter::convertToNumeric(geno_data) + + training_log <- append(training_log, + paste0( + "#Running training population", + " genotype data cleaning.\n\n" + ) + ) + + message("training_log: ", training_log) + + geno_filter_output <- genoDataFilter::filterGenoData( + geno_data, + maf = maf, + markerFilter = marker_filter_threshold, + indFilter = pheno_filter_threshold, + logReturn = TRUE + ) + + geno_data <- geno_filter_output$data + geno_filtering_log <- geno_filter_output$log + geno_data <- roundAlleleDosage(geno_data) + filtered_training_geno_data <- geno_data + +} else { + geno_filtering_log <- scan(geno_filtering_log_file, + what = "character", sep = "\n" + ) + + geno_filtering_log <- paste0(geno_filtering_log, collapse = "\n") +} + +message("geno filtering logfile: ", geno_filtering_log_file) +message("geno filtering log: ", geno_filtering_log) + +training_log <- append(training_log, geno_filtering_log) + +geno_data <- geno_data[order(row.names(geno_data)), ] + +if (length(formatted_pheno_file) != 0 && + file.info(formatted_pheno_file)$size != 0) { + formatted_pheno_data <- data.frame( + fread(formatted_pheno_file, + header = TRUE, + na.strings = c("NA", "", "--", "-", ".") + ) + ) +} else { + if (dataset_info == "combined_populations") { + pheno_file <- grep("model_phenodata", + input_files, + value = TRUE + ) + } else { + pheno_file <- grep("\\/phenotype_data", + input_files, + value = TRUE + ) + } + + if (is.null(pheno_file)) { + stop("phenotype data file is missing.") + } + + if (file.info(pheno_file)$size == 0) { + stop(paste0("phenotype data file ", pheno_file, " is empty.")) + + } + + pheno_data <- data.frame(fread(pheno_file, + sep = "\t", + na.strings = c("NA", "", "--", "-", "."), + header = TRUE)) +} + +pheno_trait_data <- c() +trait_raw_pheno_data <- c() +anova_log <- paste0("#Preprocessing training population phenotype data.\n\n") + +if (dataset_info == "combined_populations") { + anova_log <- scan(analysis_report_file, what = "character", sep = "\n") + anova_log <- paste0(anova_log, collapse = "\n") + + if (!is.null(formatted_pheno_data)) { + pheno_trait_data <- subset(formatted_pheno_data, select = trait_abbr) + pheno_trait_data <- na.omit(pheno_trait_data) + } else { + if (any(grepl("Average", names(pheno_data)))) { + pheno_trait_data <- pheno_data %>% + select(V1, Average) %>% + data.frame + } else { + pheno_trait_data <- pheno_data + } + + colnames(pheno_trait_data) <- c("genotypes", trait_abbr) + } +} else { + if (!is.null(formatted_pheno_data)) { + pheno_trait_data <- subset(formatted_pheno_data, + select = c("V1", trait_abbr)) + + pheno_trait_data <- as.data.frame(pheno_trait_data) + pheno_trait_data <- na.omit(pheno_trait_data) + colnames(pheno_trait_data)[1] <- "genotypes" + + } else if (length(grep("list", pheno_file)) != 0) { + pheno_trait_data <- phenoAnalysis::averageTrait(pheno_data, trait_abbr) + } else { + pheno_adjusted_means_result <- phenoAnalysis::getAdjMeans( + pheno_data, + traitName = trait_abbr, + calcAverages = TRUE, + logReturn = TRUE + ) + + + anova_log <- paste0(anova_log, pheno_adjusted_means_result$log) + pheno_trait_data <- pheno_adjusted_means_result$adjMeans + } + + meta_cols_kept <- c("observationUnitName", "germplasmName", + "studyDbId", "locationName", + "studyYear", "replicate", + "blockNumber", trait_abbr) + + trait_raw_pheno_data <- pheno_data %>% + select(all_of(meta_cols_kept)) + +} + +mean_type <- names(pheno_trait_data)[2] +names(pheno_trait_data) <- c("genotypes", trait_abbr) + +selection_pop_temp_file <- grep("selection_population", + input_files, value = TRUE) + +selection_pop_geno_file <- c() +filtered_selection_geno_file <- c() +selection_all_files <- c() + +if (length(selection_pop_temp_file) != 0) { + selection_all_files <- scan(selection_pop_temp_file, + what = "character") + + selection_pop_geno_file <- grep("\\/genotype_data", + selection_all_files, + value = TRUE) + + #filtered_selection_geno_file <- grep("filtered_genotype_data_", selection_all_files, value = TRUE) +} + +selection_pop_gebvs_file <- grep("rrblup_selection_gebvs", + output_files, value = TRUE) + +selection_pop_data <- c() +# read_filtered_pred_geno_data <- c() +# filtered_pred_geno_data <- c() + +## if (length(filtered_selection_geno_file) != 0 && file.info(filtered_selection_geno_file)$size != 0) { +## selection_pop_data <- fread(filtered_selection_geno_file, na.strings = c("NA", " ", "--", "-"),) +## read_filtered_pred_geno_data <- 1 + +## selection_pop_data <- data.frame(selection_pop_data) +## rownames(selection_pop_data) <- selection_pop_data[, 1] +## selection_pop_data[, 1] <- NULL + +## } else +selection_prediction_log <- c() +if (length(selection_pop_geno_file) != 0) { + selection_prediction_log <- append( + selection_prediction_log, + paste0("#Data preprocessing of selection population genotype data.\n\n") + ) + + selection_pop_data <- fread(selection_pop_geno_file, + header = TRUE, + na.strings = c("NA", "", "--", "-")) + + selection_pop_data <- data.frame(selection_pop_data) + message("clones count selection_pop_data before removing duplicates:\n", + length(rownames(selection_pop_data))) + selection_pop_data <- selection_pop_data[!duplicated(selection_pop_data$V1), ] + message("clones count selection_pop_data after removing duplicates:\n", + length(rownames(selection_pop_data))) + selection_pop_data <- selection_pop_data[!is.na(selection_pop_data$V1), ] + message("clones count selection_pop_data after removing NA:\n", + length(rownames(selection_pop_data))) + + selection_pop_data <- column_to_rownames(selection_pop_data, "V1") + + + selection_pop_data <- genoDataFilter::convertToNumeric(selection_pop_data) + + selection_prediction_log <- append( + selection_prediction_log, + paste0("Running selection population genotype data cleaning.") + ) + + selection_pop_filtered_data <- filterGenoData(selection_pop_data, + maf = maf, + markerFilter = marker_filter_threshold, + indFilter = pheno_filter_threshold, + logReturn = TRUE + ) + + selection_pop_data <- selection_pop_filtered_data$data + selection_prediction_log <- append(selection_prediction_log, + selection_pop_filtered_data$log) + + selection_pop_data <- roundAlleleDosage(selection_pop_data) +} + +#impute genotype values for obs with missing values, +geno_data_missing <- c() + +if (sum(is.na(geno_data)) > 0) { + geno_data_missing <- c("yes") + geno_data <- na.roughfix(geno_data) + geno_data <- data.frame(geno_data) +} + +#create phenotype and genotype datasets with +#common stocks only + +#extract observation lines with both +#phenotype and genotype data only. +training_log <- append( + training_log, + paste0("\n\n#Filtering for training population genotypes", + " with both phenotype and marker data.\n\n" + ) +) + +training_log <- append( + training_log, + paste0("After calculating trait averages,", + " the training population phenotype dataset has ", + length(rownames(pheno_trait_data)), + " individuals.\n" + ) +) + +training_log <- append( + training_log, + paste0("After cleaning up for missing values,", + " the training population genotype dataset has ", + length(rownames(geno_data)), + " individuals.\n" + ) +) + +common_genotypes <- intersect(pheno_trait_data$genotypes, row.names(geno_data)) + +training_log <- append( + training_log, + paste0(length(common_genotypes), + " individuals are shared in both phenotype", + " and genotype datasets.\n" + ) +) + +#remove genotyped lines without phenotype data +geno_data_filtered_genotypes <- geno_data[(rownames(geno_data) %in% + common_genotypes), ] + +training_log <- append( + training_log, + paste0("After removing individuals without phenotype data,", + " this genotype dataset has ", + length(rownames(geno_data_filtered_genotypes)), + " individuals.\n" + ) +) + +#remove phenotyped lines without genotype data +pheno_trait_data <- pheno_trait_data[(pheno_trait_data$genotypes %in% + common_genotypes), ] + +training_log <- append( + training_log, + paste0("After removing individuals without genotype data,", + " this phenotype dataset has ", + length(rownames(pheno_trait_data)), + " individuals.\n" + ) +) + +pheno_trait_for_mixed_solve <- data.frame(pheno_trait_data) +rownames(pheno_trait_for_mixed_solve) <- pheno_trait_for_mixed_solve[, 1] +pheno_trait_for_mixed_solve[, 1] <- NULL + +#impute missing data in prediction data + +selection_data_missing <- c() +if (length(selection_pop_data) != 0) { + #purge markers unique to both populations + training_pop_markers <- names(geno_data_filtered_genotypes) + selection_pop_markers <- names(selection_pop_data) + + selection_prediction_log <- append( + selection_prediction_log, + paste0("#Comparing markers in the training and", + " selection populations genotype datasets.\n\n" + ) + ) + + selection_prediction_log <- append( + selection_prediction_log, + paste0("The training population genotype dataset has ", + length(training_pop_markers), + " markers.\n" + ) + ) + + selection_prediction_log <- append( + selection_prediction_log, + paste0("The selection population genotype dataset has ", + length(selection_pop_markers), + " markers.\n" + ) + ) + + common_markers <- intersect(training_pop_markers, selection_pop_markers) + selection_prediction_log <- append( + selection_prediction_log, + paste0("The training and selection populations genotype dataset have ", + length(training_pop_markers), + " markers in common.\n" + ) + ) + + geno_data_filtered_genotypes <- subset(geno_data_filtered_genotypes, + select = common_markers) + + selection_prediction_log <- append( + selection_prediction_log, + paste0("After filtering for shared markers,", + " the training population genotype dataset has ", + length(names(selection_pop_data)), " markers.\n" + ) + ) + + selection_pop_data <- subset(selection_pop_data, select = common_markers) + selection_prediction_log <- append( + selection_prediction_log, + paste0("After filtering for shared markers,", + " the selection population genotype dataset has ", + length(names(selection_pop_data)), + " markers.\n" + ) + ) + + if (sum(is.na(selection_pop_data)) > 0) { + selection_data_missing <- c("yes") + selection_pop_data <- na.roughfix(selection_pop_data) + selection_pop_data <- data.frame(selection_pop_data) + } +} +#change genotype coding to [-1, 0, 1], to use the A.mat ) if [0, 1, 2] +genotype_encoding <- grep("2", geno_data_filtered_genotypes[1, ], value = TRUE) +if (length(genotype_encoding)) { + geno_data <- geno_data - 1 + geno_data_filtered_genotypes <- geno_data_filtered_genotypes - 1 +} + +if (length(selection_pop_data) != 0) { + genotype_encoding <- grep("2", selection_pop_data[1, ], value = TRUE) + if (length(genotype_encoding) != 0) { + selection_pop_data <- selection_pop_data - 1 + } +} + +ordered_marker_effects <- c() +training_pop_gebvs <- c() +all_validations <- c() +combined_gebvs_file <- c() +all_gebvs <- c() +model_pheno_data <- c() +kinship_matrix <- c() +training_pop_genetic_values <- c() +selection_pop_genetic_values <- c() +combined_training_gebvs_genetic_values <- c() +combined_selection_gebvs_genetic_values <- c() +trait_genetic_values_header <- paste0(trait_abbr, "_genetic_values") +#additive relationship model +#calculate the inner products for +#genotypes (realized relationship matrix) +kinship_matrix_file <- grep( + "relationship_matrix_table", + output_files, + value = TRUE +) + +kinship_matrix_json_file <- grep( + "relationship_matrix_json", + output_files, + value = TRUE +) + +trait_kinship_matrix_file <- grep( + "relationship_matrix_adjusted_table", + output_files, + value = TRUE +) + +trait_kinship_matrix_json_file <- grep( + "relationship_matrix_adjusted_json", + output_files, + value = TRUE +) + +inbreeding_file <- grep( + "inbreeding_coefficients", + output_files, + value = TRUE +) + +average_kinship_file <- grep( + "average_kinship", + output_files, + value = TRUE +) + +inbreeding <- c() +average_kinship <- c() + +if (length(kinship_matrix_file) != 0) { + if (file.info(kinship_matrix_file)$size > 0) { + kinship_matrix <- data.frame( + fread( + kinship_matrix_file, + header = TRUE) + ) + + rownames(kinship_matrix) <- kinship_matrix[, 1] + kinship_matrix[, 1] <- NULL + colnames(kinship_matrix) <- rownames(kinship_matrix) + kinship_matrix <- data.matrix(kinship_matrix) + + } else { + kinship_matrix <- A.mat(geno_data) + diag(kinship_matrix) <- diag(kinship_matrix) %>% + replace(., . < 1, 1) + kinship_matrix <- kinship_matrix %>% + replace(., . <= 0, 0.00001) + + inbreeding <- diag(kinship_matrix) + inbreeding <- inbreeding - 1 + inbreeding <- data.frame(inbreeding) + + inbreeding <- inbreeding %>% + rownames_to_column("genotypes") %>% + rename(Inbreeding = inbreeding) %>% + arrange(Inbreeding) %>% + mutate_at("Inbreeding", round, 3) %>% + column_to_rownames("genotypes") + } +} + +kinship_matrix <- data.frame(kinship_matrix) +colnames(kinship_matrix) <- rownames(kinship_matrix) + +kinship_matrix <- rownames_to_column(kinship_matrix, var = "genotypes") +kinship_matrix <- kinship_matrix %>% mutate_if(is.numeric, round, 5) +kinship_matrix <- column_to_rownames(kinship_matrix, var = "genotypes") + +trait_kinship_matrix <- kinship_matrix[(rownames(kinship_matrix) %in% + common_genotypes), ] + +trait_kinship_matrix <- trait_kinship_matrix[, + (colnames(trait_kinship_matrix) %in% + common_genotypes)] + +kinship_log <- c() +if (any(eigen(trait_kinship_matrix)$values < 0)) { + kinship_log <- paste0( + "\n\nNote: The kinship matrix of this dataset causes", + " \"Not positive semi-definite error\"", + " while running the Cholesky decomposition.", + " To fix this and run the modeling, a corrected", + " positive semi-definite matrix was computed", + " using the \"Matrix::nearPD\" function. The negative eigen values", + " from this decomposition nudged to positive values.\n\n" + ) + + trait_kinship_matrix <- Matrix::nearPD(as.matrix(trait_kinship_matrix))$mat +} + +trait_kinship_matrix <- data.matrix(trait_kinship_matrix) + +cores_count <- detectCores() + +if (cores_count > 1) { + cores_count <- (cores_count %/% 2) +} else { + cores_count <- 1 +} + +variance_components <- c() + +modeling_log <- paste0("\n\n#Training a model for ", trait_abbr, ".\n\n") +modeling_log <- append( + modeling_log, + paste0("The genomic prediction modeling follows a two-step approach.", + " First trait average values, as described above,", + " are computed for each genotype.", + " This is followed by the model fitting on the basis", + " of single phenotype value", + " for each genotype entry and kinship matrix ", + " computed from their marker data.\n" + ) +) + +if (length(kinship_log)) { + modeling_log <- append(modeling_log, paste0(kinship_log)) +} + +if (length(selection_pop_data) == 0) { + + training_model_result <- kin.blup( + data = pheno_trait_data, + geno = "genotypes", + pheno = trait_abbr, + K = trait_kinship_matrix, + n.core = cores_count, + PEV = TRUE + ) + + modeling_log <- paste0(modeling_log, + "\nThe model training is based on rrBLUP R package, version ", + packageVersion("rrBLUP"), + ". GEBVs are predicted using the kin.blup function", + " and GBLUP method.\n\n" + ) + + training_pop_genetic_values <- data.frame( + round(training_model_result$pred, 2) + ) + + colnames(training_pop_genetic_values) <- trait_genetic_values_header + training_pop_genetic_values <- training_pop_genetic_values %>% + arrange(across(trait_genetic_values_header, desc)) + + training_pop_genetic_values <- rownames_to_column( + training_pop_genetic_values, + var = "genotypes" + ) + + training_pop_gebvs <- training_model_result$g + training_gebv_pev <- training_model_result$PEV + training_gebv_stderr <- sqrt(training_gebv_pev) + training_gebv_stderr <- data.frame(round(training_gebv_stderr, 2)) + + training_pop_gebvs <- data.frame(round(training_pop_gebvs, 2)) + + colnames(training_gebv_stderr) <- c("SE") + colnames(training_pop_gebvs) <- trait_abbr + + training_gebv_stderr <- rownames_to_column(training_gebv_stderr, + var = "genotypes") + + training_pop_gebvs <- rownames_to_column(training_pop_gebvs, var = "genotypes") + + training_gebv_stderr <- full_join(training_pop_gebvs, training_gebv_stderr) + + + + training_gebv_stderr <- training_gebv_stderr %>% + arrange(across(trait_abbr, desc)) + + training_gebv_stderr <- column_to_rownames(training_gebv_stderr, + var = "genotypes") + + training_pop_gebvs <- training_pop_gebvs %>% arrange(across(trait_abbr, desc)) + + if (!is.null(training_pop_genetic_values) && + !is.null(training_pop_gebvs)) { + combined_training_gebvs_genetic_values <- inner_join( + training_pop_gebvs, + training_pop_genetic_values, + by = 'genotypes' + ) + } + + combined_training_gebvs_genetic_values <- column_to_rownames( + combined_training_gebvs_genetic_values, + var = "genotypes" + ) + + training_pop_genetic_values <- column_to_rownames( + training_pop_genetic_values, + var = "genotypes" + ) + + training_pop_gebvs <- column_to_rownames( + training_pop_gebvs, + var = "genotypes" + ) + + pheno_trait_for_mixed_solve <- data.matrix(pheno_trait_for_mixed_solve) + geno_data_filtered_genotypes <- data.matrix(geno_data_filtered_genotypes) + + mixed_solve_output <- mixed.solve( + y = pheno_trait_for_mixed_solve, + Z = geno_data_filtered_genotypes + ) + + ordered_marker_effects <- data.matrix(mixed_solve_output$u) + ordered_marker_effects <- data.matrix( + ordered_marker_effects [order(-ordered_marker_effects[, 1]), ] + ) + ordered_marker_effects <- round(ordered_marker_effects, 5) + + colnames(ordered_marker_effects) <- c("Marker Effects") + ordered_marker_effects <- data.frame(ordered_marker_effects) + + model_pheno_data <- data.frame(round(pheno_trait_for_mixed_solve, 2)) + + heritability <- round(( + training_model_result$Vg / (training_model_result$Ve + + training_model_result$Vg)), 2) + + additive_variance <- round(training_model_result$Vg, 2) + error_variance <- round(training_model_result$Ve, 2) + + variance_components <- c("\nAdditive genetic variance\t", + additive_variance, "\n") + variance_components <- append(variance_components, + c("Error variance\t", + error_variance, "\n") + ) + + variance_components <- append(variance_components, + c("SNP heritability (h)\t", + heritability, "\n")) + + combined_gebvs_file <- grep( + "selected_traits_gebv", + output_files, + ignore.case = TRUE, + value = TRUE + ) + + if (length(combined_gebvs_file) != 0) { + file_size <- file.info(combined_gebvs_file)$size + if (file_size != 0) { + combined_gebvs <- data.frame(fread(combined_gebvs_file, + header = TRUE)) + + rownames(combined_gebvs) <- combined_gebvs[, 1] + combined_gebvs[, 1] <- NULL + + all_gebvs <- merge(combined_gebvs, training_pop_gebvs, + by = 0, + all = TRUE + ) + + rownames(all_gebvs) <- all_gebvs[, 1] + all_gebvs[, 1] <- NULL + } + } + + #cross-validation + + if (is.null(selection_pop_geno_file)) { + genotypes_count <- nrow(pheno_trait_data) + + if (genotypes_count < 20 ) { + warning(genotypes_count, " is too small number of genotypes.") + } + + set.seed(4567) + + k <- 10 + reps <- 2 + cross_val_folds <- createMultiFolds(pheno_trait_data[, 2], + k = k, times = reps) + + modeling_log <- paste0( + modeling_log, + "Model prediction accuracy is evaluated using", + " cross-validation method. ", + k, + " folds, replicated ", + reps, + " times are used to predict the model accuracy.\n\n" + ) + + for (rep in 1:reps) { + rep_name <- paste0("Rep", rep) + + for (i in 1:k) { + validation_group_name <- ifelse(i < 10, "Fold0", "Fold") + + validation_training_rep_name <- paste0(validation_group_name, + i, ".", rep_name) + + validation_training_clones <- cross_val_folds[[validation_training_rep_name]] + validation_selection_clones <- as.numeric( + rownames( + pheno_trait_data[-validation_training_clones, ] + ) + ) + + kblup <- paste("rKblup", i, sep = ".") + + validation_prediction_result <- kin.blup( + data = pheno_trait_data[validation_training_clones, ], + geno = "genotypes", + pheno = trait_abbr, + K = trait_kinship_matrix, + n.core = cores_count, + PEV = TRUE + ) + + assign(kblup, validation_prediction_result) + + #calculate cross-validation accuracy + validation_group_gebvs <- validation_prediction_result$g + validation_group_gebvs <- data.frame(validation_group_gebvs) + + validation_selection_clones <- validation_selection_clones[ + which(validation_selection_clones <= nrow(pheno_trait_data)) + ] + + validation_selection_geno_data <- pheno_trait_data[( + rownames(pheno_trait_data) %in% + validation_selection_clones), ] + + rownames(validation_selection_geno_data) <- validation_selection_geno_data[, 1] + validation_selection_geno_data[, 1] <- NULL + + validation_group_gebvs <- rownames_to_column(validation_group_gebvs, + var = "genotypes") + + validation_selection_geno_data <- rownames_to_column( + validation_selection_geno_data, + var = "genotypes" + ) + + validation_corr_data <- inner_join( + validation_selection_geno_data, + validation_group_gebvs, + by = "genotypes" + ) + + validation_corr_data$genotypes <- NULL + + accuracy <- try(cor(validation_corr_data)) + validation <- paste("validation", + validation_training_rep_name, + sep = "." + ) + + cross_validation_name <- paste("CV", + validation_training_rep_name, sep = " " + ) + + if (inherits(accuracy, "try-error") == FALSE) { + accuracy <- round(accuracy[1, 2], digits = 3) + accuracy <- data.matrix(accuracy) + + colnames(accuracy) <- c("correlation") + rownames(accuracy) <- cross_validation_name + + assign(validation, accuracy) + + if (!is.na(accuracy[1, 1])) { + all_validations <- rbind(all_validations, accuracy) + } + } + } + } + + all_validations <- data.frame( + all_validations[order(-all_validations[, 1]), ] + ) + colnames(all_validations) <- c("Correlation") + } +} + +selection_prediction_result <- c() +selection_pop_gebvs <- c() +selection_pop_gebvs_stderr <- c() +selection_pop_genetic_values <- c() + +#selection pop geno data after cleaning up +#and removing unique markers to selection pop + +filtered_sel_geno_data <- selection_pop_data +if (length(selection_pop_data) != 0) { + combined_training_selection_geno_data <- rbind(geno_data_filtered_genotypes, + selection_pop_data + ) + kinship_combined_training_selection_pop <- A.mat(combined_training_selection_geno_data) + + selection_prediction_result <- kin.blup( + data = pheno_trait_data, + geno = "genotypes", + pheno = trait_abbr, + K = kinship_combined_training_selection_pop, + n.core = cores_count, + PEV = TRUE + ) + + selection_pop_genotypes <- rownames(selection_pop_data) + + selection_pop_genetic_values <- data.frame( + round(selection_prediction_result$pred, 2) + ) + + colnames(selection_pop_genetic_values) <- trait_genetic_values_header + selection_pop_genetic_values <- rownames_to_column( + selection_pop_genetic_values, + var = "genotypes" + ) + + selection_pop_genetic_values <- selection_pop_genetic_values %>% + filter(genotypes %in% + selection_pop_genotypes) + + selection_pop_genetic_values <- selection_pop_genetic_values %>% + filter(genotypes %in% + selection_pop_genotypes) + + selection_pop_genetic_values <- selection_pop_genetic_values %>% + arrange(across(all_of(trait_genetic_values_header), desc)) + + selection_pop_gebvs <- round(data.frame(selection_prediction_result$g), 2) + colnames(selection_pop_gebvs) <- trait_abbr + selection_pop_gebvs <- rownames_to_column( + selection_pop_gebvs, + var = "genotypes" + ) + + selection_pop_gebvs <- selection_pop_gebvs %>% + filter(genotypes %in% + selection_pop_genotypes) + + # sortVar <- parse_expr(trait_abbr) + + selection_pop_pev <- selection_prediction_result$PEV + selection_pop_stderr <- sqrt(selection_pop_pev) + selection_pop_stderr <- data.frame(round(selection_pop_stderr, 2)) + colnames(selection_pop_stderr) <- "SE" + selection_pop_stderr <- rownames_to_column( + selection_pop_stderr, + var = "genotypes" + ) + selection_pop_stderr <- selection_pop_stderr %>% + filter(genotypes %in% + selection_pop_genotypes) + + selection_pop_gebvs_stderr <- inner_join( + selection_pop_gebvs, + selection_pop_stderr, + by = "genotypes" + ) + + selection_pop_gebvs <- selection_pop_gebvs %>% + arrange(across(all_of(trait_abbr), desc)) + + + if (!is.null(selection_pop_genetic_values) && + !is.null(selection_pop_gebvs)) { + combined_selection_gebvs_genetic_values <- inner_join(selection_pop_gebvs, + selection_pop_genetic_values, + by = 'genotypes') + } + + combined_selection_gebvs_genetic_values <- column_to_rownames( + combined_selection_gebvs_genetic_values, + var = "genotypes" + ) + + selection_pop_genetic_values <- column_to_rownames( + selection_pop_genetic_values, + var = "genotypes" + ) + + selection_pop_gebvs <- column_to_rownames( + selection_pop_gebvs, + var = "genotypes" + ) + + selection_pop_gebvs_stderr <- selection_pop_gebvs_stderr %>% + arrange(across(all_of(trait_abbr), desc)) + + selection_pop_gebvs_stderr <- column_to_rownames( + selection_pop_gebvs_stderr, + var = "genotypes" + ) +} + + +if (!is.null(selection_pop_gebvs) && length(selection_pop_gebvs_file) != 0) { + fwrite( + selection_pop_gebvs, + file = selection_pop_gebvs_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(training_pop_genetic_values) && + length(training_pop_genetic_values_file) != 0) { + fwrite( + training_pop_genetic_values, + file = training_pop_genetic_values_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + + fwrite( + combined_training_gebvs_genetic_values, + file = combined_training_gebvs_genetic_values_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + + + +if (!is.null(selection_pop_genetic_values) && + length(selection_pop_genetic_values_file) != 0) { + fwrite( + selection_pop_genetic_values, + file = selection_pop_genetic_values_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + + fwrite( + combined_selection_gebvs_genetic_values, + file = combined_selection_gebvs_genetic_values_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(all_validations)) { + fwrite( + all_validations, + file = validation_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(ordered_marker_effects)) { + fwrite( + ordered_marker_effects, + file = marker_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + + +if (!is.null(training_pop_gebvs)) { + fwrite( + training_pop_gebvs, + file = blup_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (length(combined_gebvs_file) != 0 ) { + if (file.info(combined_gebvs_file)$size == 0) { + fwrite( + training_pop_gebvs, + file = combined_gebvs_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + } else { + fwrite( + all_gebvs, + file = combined_gebvs_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + } +} + +if (!is.null(model_pheno_data) && + length(model_pheno_file) != 0) { + + if (!is.null(mean_type)) { + colnames(model_pheno_data) <- mean_type + } + + fwrite( + model_pheno_data, + file = model_pheno_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(geno_data_filtered_genotypes) && + length(model_geno_file) != 0) { + + fwrite( + geno_data_filtered_genotypes, + file = model_geno_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(trait_raw_pheno_data) && + length(trait_raw_pheno_file) != 0) { + + fwrite( + trait_raw_pheno_data, + file = trait_raw_pheno_file, + row.names = FALSE, + sep = "\t", + na = "NA", + quote = FALSE, + ) +} + +if (!is.null(filtered_training_geno_data) && + file.info(filtered_training_geno_file)$size == 0) { + fwrite( + filtered_training_geno_data, + file = filtered_training_geno_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + + cat(geno_filtering_log, + fill = TRUE, + file = geno_filtering_log_file, + append = FALSE + ) +} + +if (length(filtered_sel_geno_file) != 0 && + file.info(filtered_sel_geno_file)$size == 0) { + fwrite( + filtered_sel_geno_data, + file = filtered_sel_geno_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +## if (!is.null(geno_data_missing)) { +## write.table(geno_data, +## file = geno_file, +## sep = "\t", +## col.names = NA, +## quote = FALSE, +## ) + +## } + +## if (!is.null(predictionDataMissing)) { +## write.table(predictionData, +## file = predictionFile, +## sep = "\t", +## col.names = NA, +## quote = FALSE, +## ) +## } + +if (file.info(kinship_matrix_file)$size == 0) { + + fwrite(kinship_matrix, + file = kinship_matrix_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (file.info(kinship_matrix_json_file)$size == 0) { + + kinship_matrix_json <- kinship_matrix + kinship_matrix_json[upper.tri(kinship_matrix_json)] <- NA + + + kinship_matrix_json <- data.frame(kinship_matrix_json) + + kinship_matrix_list <- list(labels = names(kinship_matrix_json), + values = kinship_matrix_json) + + kinship_matrix_json <- jsonlite::toJSON(kinship_matrix_list) + + + write(kinship_matrix_json, + file = kinship_matrix_json_file, + ) +} + +if (file.info(trait_kinship_matrix_file)$size == 0) { + + inbre <- diag(trait_kinship_matrix) + inbre <- inbre - 1 + + diag(trait_kinship_matrix) <- inbre + + trait_kinship_matrix <- data.frame(trait_kinship_matrix) %>% + replace(., . < 0, 0) + + fwrite(trait_kinship_matrix, + file = trait_kinship_matrix_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) + + if (file.info(trait_kinship_matrix_json_file)$size == 0) { + + trait_kinship_matrix_json <- trait_kinship_matrix + trait_kinship_matrix_json[upper.tri(trait_kinship_matrix_json)] <- NA + + trait_kinship_matrix_json <- data.frame(trait_kinship_matrix_json) + + trait_kinship_matrix_list <- list( + labels = names(trait_kinship_matrix_json), + values = trait_kinship_matrix_json + ) + + trait_kinship_matrix_json <- jsonlite::toJSON(trait_kinship_matrix_list) + + write(trait_kinship_matrix_json, + file = trait_kinship_matrix_json_file, + ) + } +} + + +if (file.info(inbreeding_file)$size == 0) { + fwrite(inbreeding, + file = inbreeding_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (file.info(average_kinship_file)$size == 0) { + average_kinship <- data.frame(apply(trait_kinship_matrix, 1, mean)) + + average_kinship <- average_kinship %>% + rownames_to_column("genotypes") %>% + rename(Mean_kinship = contains("apply.trait_kinship_matrix")) %>% + arrange(Mean_kinship) %>% + mutate_at("Mean_kinship", round, 3) %>% + column_to_rownames("genotypes") + + fwrite(average_kinship, + file = average_kinship_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (file.info(formatted_pheno_file)$size == 0 && + !is.null(formatted_pheno_data)) { + fwrite(formatted_pheno_data, + file = formatted_pheno_file, + row.names = TRUE, + sep = "\t", + quote = FALSE, + ) +} + +if (!is.null(variance_components)) { + cat(variance_components, + file = variance_components_file + ) +} + +if (!is.null(selection_prediction_log)) { + cat(log_heading, + selection_prediction_log, + fill = TRUE, + file = analysis_report_file, + append = FALSE + ) +} else { + cat(log_heading, + anova_log, + training_log, + modeling_log, + fill = TRUE, + file = analysis_report_file, + append = FALSE + ) +} + +message("Done.") + +q(save = "no", runLast = FALSE) \ No newline at end of file diff --git a/R/spatial_correlation_check.R b/R/spatial_correlation_check.R new file mode 100644 index 0000000000..a815f3acce --- /dev/null +++ b/R/spatial_correlation_check.R @@ -0,0 +1,240 @@ +################################################################################ +# Checking for spatial correlation in the phenotypic data +################################################################################ + +# There are 6 main steps to this protocol: +# 1. Load the software needed. +# 2. Declare user-supplied variables. +# 3. Process the phenotypic data +# 4. Loop through the traits and check fitst for quality then spatial correlation +# 5. Format the information needed for output +# 6. Save the output + + +################################################################################ +# 1. Load software needed +################################################################################ +# Check installation of required packages +# if (!require("dplyr")) { +# install.packages("dplyr") +# } +# if (!require("spdep")) { +# install.packages("sf") +# install.packages("spdep", dependencies = TRUE) +# } +# if (!require("gstat")) { +# install.packages("gstat", dependencies = TRUE) +# } +# # if (!require("raster")) { +# # install.packages("raster") +# # } +# if (!require("ggplot2")) { +# install.packages("ggplot2") +# } +# if (!require("reshape2")) { +# install.packages("reshape2") +# } +# if (!require("moments")) { +# install.packages("moments") +# } +# if (!require("stats")) { +# install.packages("stats") +# } +################################################################### +# Load libraries +################################################################### +library('dplyr') +library('spdep') +library('gstat') +# library(raster) +library('ggplot2') +library('reshape2') +library('moments') +library('stats') +################################################################### +# 2. Declare user-supplied variables. +################################################################### +# Get Arguments +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Two or more arguments are required.") +} +phenotypeFile <- args[1] +traits <- args[2] +replicate <- "replicate" +################################################################### +# 3. Process the phenotypic data +################################################################### +# read in the phenotypic data +userPheno <- read.delim(phenotypeFile, header = TRUE, sep = "\t", fill = TRUE, check.names = FALSE) +# and the traits +userResponse <- unlist(strsplit(traits, split = ",", fixed = T)) +userResponse <- userResponse[!userResponse == "notes"] # x[ !x == 'A'] # remove notes from userResponse +userResponse <- userResponse[!userResponse == "X50_sprout_emergence_time_estimation_in_yy_mm_dd_CO_343_0000201"] # x[ !x == 'A'] +# write(paste("userResponse:", userResponse), stderr()) +################################################################### +# 4. Function for checking quality of data +################################################################### +check_quality <- function(data, phenotype, replicate) { + ################################################################### + # Check the percentage of missing data + ################################################################### + trait_vals <- data[[phenotype]] + # print(trait_vals) + nas <- sum(is.na(trait_vals)) / length(trait_vals) * 100 + print(nas) + ################################################################### + # Check the uniqueness of the data + ################################################################### + unique_vals <- unique(trait_vals, na.rm = TRUE) + unique_count <- length(unique_vals) + ################################################################### + # Check the distribution and normality of phenotypic data + ################################################################### + no_na <- trait_vals[!is.na(trait_vals)] + write(paste("no_na p-value: ", no_na), stderr()) + write(paste("no_na??: ", class(no_na)), stderr()) + if (!is.numeric(no_na)) { + message("Data not numeric") + skewness <- NA + shapiro_val <- data.frame(p.value = NA) + shapiro_val$p.value <- NA + return ( summary_table <- data.frame( + phenotype = phenotype, + missing_data = nas, + skewness = skewness, + shapiro_val = NA, + outliers = 0, + unique_count = 0, + replicate_count = 0)) + } else if (length(no_na) < 3) { + message("Not enough data to check for normality") + skewness <- NA + shapiro_val <- data.frame(p.value = NA) + shapiro_val$p.value <- NA + } else if (unique_count < 3) { + message("Not enough unique data to check for normality") + skewness <- NA + shapiro_val <- data.frame(p.value = NA) + shapiro_val$p.value <- NA + } else { + skewness <- skewness(no_na) # install.packages("moments") + shapiro_val <- shapiro.test(no_na) # install.packages("stats") + } + ################################################################### + # Check the presence of outliers + ################################################################### + outliers <- sum(no_na > (mean(no_na) + 3 * sd(no_na)) | no_na < (mean(no_na) - 3 * sd(no_na))) + + ################################################################### + # Check for replicated data + ################################################################### + unique_reps <- unique(data$replicate, na.rm = TRUE) + replicate_count <- length(unique_reps) + ################################################################### + # Create a table to summarize the results + ################################################################### + summary_table <- data.frame( + phenotype = phenotype, + missing_data = nas, + skewness = skewness, + shapiro_val = shapiro_val$p.value, + outliers = outliers, + unique_count = unique_count, + replicate_count = replicate_count + ) + ################################################################### + # return the summary table + ################################################################### + return(summary_table) + ################################################################### +} +################################################################### +# 5. Loop through the traits and check fitst for quality then spatial correlation +################################################################### +output <- data.frame(quality = character(0), Moran_pvalue = numeric(0), spatial_correction_needed = character(0), trait = character(0)) + +for (i in 1:length(userResponse)) { + quality_summary <- check_quality(userPheno, userResponse[i], replicate) + # write(paste("quality_summary:", quality_summary), stderr()) + if (quality_summary$missing_data > 90) { + quality <- "too much missing data" + Moran_pvalue <- NA + spatial_correction_needed <- "NO" + } else if (quality_summary$outliers > 10) { + quality <- "too many outliers" + Moran_pvalue <- NA + spatial_correction_needed <- "NO" + } else if (is.na(quality_summary$skewness) && is.na(quality_summary$shapiro_val)) { + quality <- "not enough data to check for normality" + Moran_pvalue <- NA + spatial_correction_needed <- "NO" + } else if (quality_summary$unique_count < 2) { + quality <- "not enough unique values" + Moran_pvalue <- NA + spatial_correction_needed <- "NO" + } else if (quality_summary$replicate_count < 2) { + quality <- "not enough replicates" + Moran_pvalue <- NA + spatial_correction_needed <- "NO" + } else { + write(paste("Trait: ", userResponse[i]), stderr()) + if (quality_summary$shapiro_val < 0.05) { + quality <- "good but not normally distributed" + spatial_correction_needed <- "YES" + } else if (quality_summary$skewness > 1 || quality_summary$skewness < -1) { + quality <- "good but skewed" + # Moran_pvalue <- NA + spatial_correction_needed <- "YES" + } else { + quality <- "good data" + spatial_correction_needed <- "YES" + } + + ################################################################### + # 4.2 Check for spatial correlation + ################################################################### + rowNumber <- "rowNumber" + colNumber <- "colNumber" + # obtain rows in userPheno that are not NA in the userResponse column + userPheno_subset <- userPheno[complete.cases(userPheno[[userResponse[i]]]), ] + # data <- data[complete.cases(data[[trait]]), ] + # print str(userPheno_subset) to stderr() + # write(paste("userPheno_subset:", userPheno_subset), stderr()) + # get the userResponse column and store it in trait_vals + trait_vals <- userPheno_subset[[userResponse[i]]] + # write(paste("Trait values: ", trait_vals), stderr()) + coordinates <- userPheno_subset[, c(rowNumber, colNumber), drop = FALSE] + write(paste("Coordinates: ", coordinates), stderr()) + k <- 3 # Set the value of k + + # Check if there are enough data points + num_data_points <- nrow(coordinates) + write(paste("Number of data points: ", num_data_points), stderr()) + if (num_data_points >= k) { + kn <- knearneigh(coordinates, k = k) + # Continue with further processing using kn + nb <- knn2nb(kn) + weights <- nb2listw(nb) + + # complete_cases <- !is.na(trait_vals) + # trait_vals <- trait_vals[complete_cases] + moran <- moran.test(trait_vals, weights) + write(paste("Moran p-value: ", moran$p.value), stderr()) + Moran_pvalue <- round(moran$p.value, 5) + spatial_correction_needed <- ifelse(moran$p.value > 0.05, "NO", "YES") + } else { + # Handle the case when there are not enough data points + message("There are fewer data points than k.") + } + } + + output <- rbind(output, data.frame(trait = userResponse[i], quality, Moran_pvalue, spatial_correction_needed)) +} +write(paste("Spatial corr output: ", output), stderr()) +################################################################### +# 6. Output a table with column indicating if there is spatial correlation or not and quality of data +################################################################### +outfile <- paste(phenotypeFile, ".spatial_correlation_summary", sep = "") +write.table(output, file = outfile, sep = "\t", row.names = FALSE) +################################################################### diff --git a/R/spatial_modeling.R b/R/spatial_modeling.R index 7bbd9c98d3..462b57bc8c 100644 --- a/R/spatial_modeling.R +++ b/R/spatial_modeling.R @@ -6,7 +6,7 @@ # 1. Load the software needed. # 2. Declare user-supplied variables. # 3. Process the phenotypic data. -# 4. Fit the mixed models in sommer. +# 4. Fit the two models with and without 2D Spline model in sommer # 5. Format the information needed for output. @@ -16,84 +16,198 @@ library(sommer) - ################################################################################ # 2. Declare user-supplied variables. ################################################################################ -#Get Arguments -args = commandArgs(trailingOnly=TRUE) +# Get Arguments +args <- commandArgs(trailingOnly = TRUE) if (length(args) < 2) { - stop('Two or more arguments are required.') + stop("Two or more arguments are required.") } -phenotypeFile = args[1] -traits = args[2] - +phenotypeFile <- args[1] +traits <- args[2] ################################################################################ # 3. Process the phenotypic data. ################################################################################ -#read in the phenotypic data -userPheno <- read.delim(phenotypeFile, header = TRUE, sep="\t", fill=TRUE) +# read in the phenotypic data + +userPheno <- read.delim(phenotypeFile, header = TRUE, sep = "\t", fill = TRUE) -#The user should be able to select their response variables from a drop-down menu +# The user should be able to select their response variables from a drop-down menu # of the column names of the userPheno object. Then, those strings should be passed # to this vector, 'userResponse'. - -userResponse <- unlist(strsplit(traits, split=",", fixed=T)) +userResponse <- unlist(strsplit(traits, split = ",", fixed = T)) +userResponse <- userResponse[!userResponse == "notes"] # x[ !x == 'A'] # remove notes from userResponse +replicate <- "replicate" userID <- "germplasmName" row <- "rowNumber" col <- "colNumber" userPheno$R <- as.factor(userPheno$rowNumber) userPheno$C <- as.factor(userPheno$colNumber) - ################################################################################ -# 4. Fit the 2D Spline model in sommer +# 4. Fit the two models with and without 2D Spline model in sommer ################################################################################ # Make a list to save the models. - -userModels <- list() - -for(i in 1:length(userResponse)){ - - fixedArg <- paste(userResponse[i], " ~ ", "1 +", userID,")", sep = "") - - randArg <- paste("~vs(", R, ")+vs(", C, ")+ spl2Da(",col,"," ,row ,")", sep = "") - - - m2.sommer <- mmer(fixed = as.formula(fixedArg), - random = as.formula(randArg), - rcov= ~units, - data=userPheno, verbose = FALSE) - - - - - userModels[[i]] <- m2.sommer - +userModelsWith <- list() +userModelsWithout <- list() + +# Make lists to save the outputs for AIC, blues and fitted values +AIC_output <- data.frame() +output <- data.frame() +fitted_output <- data.frame() + +# Loop through the response variables running the model with and without spatial variation +for (i in 2:length(userResponse)) { + write(paste("userResponse:", userResponse[i]), stderr()) + # we want response variables with nas < 90% and more than one unique value + nas <- colMeans(is.na(userPheno[userResponse[i]])) * 100 + standarddev <- sapply(userPheno[userResponse[i]], sd) # not used + minimum <- min(userPheno[userResponse[i]], na.rm = TRUE) + maximum <- max(userPheno[userResponse[i]], na.rm = TRUE) + + # so if nas < 90% and there is more than one unique value, run the model + if (nas < 90 && minimum != maximum) { + write(paste("userResponse again:", userResponse[i]), stderr()) + write(paste("Percentage of Nas:", nas), stderr()) + write(paste("userResponsedata:", userPheno[userResponse[i]]), stderr()) + + ## Fit the model without spatial variation ## + modArg <- paste(userResponse[i], " ~ ", userID, " + ", replicate, sep = "") + write(paste("modArg:", modArg), stderr()) + mod <- mmer( + as.formula(modArg), + data = userPheno, + verbose = FALSE + ) + + write(paste("modArg:", modArg), stderr()) + userModelsWithout[[i]] <- mod + + ## Fit the model with spatial variation ## + + # create a formula for the fixed effects + fixedArg <- paste(userResponse[i], " ~ ", "1 +", userID, sep = "") + write(paste("fixedArg:", fixedArg), stderr()) + # create a formula for the random effects + randArg <- paste("~vsr(R)+vsr(C)+ spl2Da(", col, ",", row, ")", sep = "") + write(paste("randArg:", randArg), stderr()) + # the model + m2.sommer <- mmer( + fixed = as.formula(fixedArg), + random = as.formula(randArg), + rcov = ~units, + data = userPheno, verbose = FALSE + ) + + userModelsWith[[i]] <- m2.sommer + + ## Obtaining the results from the models: AIC, blues and fitted values ## + # AIC + AIC.without <- summary(mod)$logo$AIC + AIC.with <- summary(m2.sommer)$logo$AIC + # check if the model with spatial variation is better than the model without spatial variation by difference of 5 in AIC + if (AIC.with < AIC.without - 5) { + AIC_output <- rbind(AIC_output, data.frame(Trait = userResponse[i], AIC.without, AIC.with, Spatial.Importance = "Yes")) + } else if (AIC.with < AIC.without) { + AIC_output <- rbind(AIC_output, data.frame(Trait = userResponse[i], AIC.without, AIC.with, Spatial.Importance = "Maybe")) + } else { + AIC_output <- rbind(AIC_output, data.frame(Trait = userResponse[i], AIC.without, AIC.with, Spatial.Importance = "No")) + } + + # blues + blue <- summary(m2.sommer)$betas + output <- rbind(output, blue) + + # fitted values + fittedvals <- fitted(m2.sommer) + colname_fitted <- paste0(userResponse[i], ".fitted") + fitted_tab <- fittedvals$dataWithFitted[, c(userID, userResponse[i], colname_fitted)] # table with userID, trait values, and fitted values + colnames(fitted_tab) <- c("Name", "Phenotype_values", "Fitted_values") + fitted_tab$Phenotype_values <- round(fitted_tab$Phenotype_values, 2) + fitted_tab$Fitted_values <- round(fitted_tab$Fitted_values, 2) + # place userResponse[i] in first column + fitted_tab <- data.frame(Trait = userResponse[i], fitted_tab) + # fitted_tab <- data.frame(plotNumber = userPheno$plotNumber, fitted_tab) + write(paste("colnames:", colnames(fitted_tab)), stderr()) + fitted_output <- rbind(fitted_output, fitted_tab) + } } +################################################################################ +# 5. Write the results to files. +################################################################################ +### deal with AIC output +outfile_AIC <- paste(phenotypeFile, ".AIC", sep = "") +write.table(AIC_output, outfile_AIC) + +## dealing with blues output +print(colnames(output)) +colnames(output) <- c("ID", "Trait", "Name", "Estimate", "Std.Error") +BLUE <- as.data.frame(output) +outfile_blue <- paste(phenotypeFile, ".blues", sep = "") +write.table(BLUE, outfile_blue) + +### deal with fitted values output +write(paste("colnames fitted:", colnames(fitted_output)), stderr()) +write(paste("fitted_output:", head(fitted_output)), stderr()) +outfile_fitted <- paste(phenotypeFile, ".fitted", sep = "") +write.table(fitted_output, outfile_fitted) + + ################################################################################ # 5. Format the information needed for output. ################################################################################ -for(i in 1:length(userModels)){ - - m2.sommer <- userModels[[i]] - - #Variance_Components <- summary(m2.sommer)$varcomp - - # outputFile= paste(userID, " Spatial Variance Components", ".out", sep="") - - write.csv(Variance_Components, outputFile) - - res <- (randef(m2.sommer)$`u:variety`) - BLUP <- as.data.frame(res) - - # adj = coef(m2.sommer)$Trait - outfile_blup = paste(phenotypeFile, ".BLUPs", sep=""); - write.table(BLUP, outfile_blup) - -} \ No newline at end of file +# for (i in 1:length(userModels)) { +# m2.sommer <- userModels[[i]] +# # write(paste("model:", head(m2.sommer)), stderr()) +# m3.sommer <- data.frame(matrix(unlist(m2.sommer), nrow = length(m2.sommer), byrow = TRUE)) +# write(paste("model:", head(m3.sommer)), stderr()) +# outfile_model <- paste(phenotypeFile, ".model", sep = "") +# write.table(m3.sommer, outfile_model) + +# # m2.sommer<-as.data.frame(m2.sommer) +# # blue = summary(m2.sommer)$beta +# # BLUE<-as.data.frame(blue) +# # write(paste('BLUE:', BLUE), stderr()) + +# # # adj = coef(m2.sommer)$Trait +# # outfile_blue = paste(phenotypeFile, ".BLUEs", sep=""); +# # write.table(BLUE, outfile_blue) +# } + + +# unique_ID <- unique(userPheno$germplasmName) # create vector of unique userID +# unique_row <- unique(userPheno$rowNumber) # create vector of unique row +# unique_col <- unique(userPheno$colNumber) # create vector of unique col +# df <- list(unique_ID, unique_row, unique_col) # create list of unique userID, row, and col +# # write(paste("df:", df), stderr()) + +# newdata <- expand.grid(... = df) # create data frame of all combinations of userID, row, and col +# names(newdata) <- c("userID", "row", "col") # name columns of new data frame + +# # write(paste("newdata:", newdata), stderr()) +# R_matrix <- diag(length(unique(userPheno$R))) # create diagonal matrix of length unique R +# C_matrix <- diag(length(unique(userPheno$C))) # create diagonal matrix of length unique C +# Z <- kronecker(R_matrix, C_matrix) # kronecker product of R and C + +# num_combinations <- nrow(newdata) # number of combinations of userID, row, and col +# # write(paste("num_combinations:", num_combinations), stderr()) +# # Z_list <- list(Z) +# Z_list <- vector("list", num_combinations) # create list of length num_combinations +# for (i in 1:num_combinations) { +# Z_list[[i]] <- Z # fill list with Z +# } +# Z <- do.call(rbind, Z_list) # combine list into matrix + +# newdata$spl2Da <- with(newdata, spl2Da(col, row)) # add splines to new data frame +# newdata[, colnames(Z)] <- Z # add random effects matrix to new data frame +# # write(paste("newdata:", newdata), stderr()) + +# # adj_means <- predict(m2.sommer, newdata, type = "response") +# adj_means <- predict.mmer(m2.sommer, classify = userID) +# write(paste("adj_means:", adj_means), stderr()) diff --git a/R/stability/ammi_script.R b/R/stability/ammi_script.R index 26b46ef5e6..529d6103bf 100644 --- a/R/stability/ammi_script.R +++ b/R/stability/ammi_script.R @@ -1,139 +1,354 @@ - - library("methods") -library("dplyr") -library("gridExtra") library("agricolae") -library("gge") +library("dplyr") +library("tidyr") -##### Get data ##### -args = commandArgs(trailingOnly = TRUE) +set.seed(54326) + +##### Getting data ##### +args = commandArgs(trailingOnly = TRUE) pheno <- read.table(args[1], sep = "\t", header = TRUE) study_trait <- args[2] -cat("Study trait is ", study_trait[1]) -figure1_file_name <- args[3] -figure2_file_name <- args[4] -AMMIFile <- args[5] -method <- args[6] +imputPheno <- args[3] +stability_method <- args[4] +jsonFile <- args[5] +graphFile <- args[6] +messageFile <- args[7] +jsonSummary <- args[8] + + +study_trait <- gsub("\\."," ",study_trait) #Making names standard names <- colnames(pheno) new_names <- gsub(".CO.*","", names) +new_names <- gsub("\\."," ",new_names) colnames(pheno) <- new_names -cat(colnames(pheno),"\n") - -#Finding which column is the study trait -for (i in 1:ncol(pheno)){ - a = noquote(colnames(pheno[i])) - b = study_trait - cat("COL: ", a, " vs study_trait: ", b,"\n") - if (a==b){ - print(a) - col = i - i = ncol(pheno) - break - }else{ - cat("working ",i,"\n") - i=i+1 - } + +## Function for pheno imputation +pheno_imputation <- function(inData){ + library(mice) + + dat_imput <- spread(inData, Env, Yield) + colnames(dat_imput) <- gsub(" ", "", colnames(dat_imput)) + #pmm is is the predictive mean matching + pre_imputed <- mice(dat_imput, method = "mean", m = 3, maxit = 10) + final_imputed <- complete(pre_imputed, 1) + + final_imputed <- gather(final_imputed, "Env", "Yield", -c("Gen","Rep")) + cat("Imput ok!") + return(final_imputed) } -env <-as.factor(pheno$locationName) -gen <-as.factor(pheno$germplasmName) -rep <-as.factor(pheno$replicate) -# trait <-as.numeric(pheno[,col]) - -message<-"The analysis could not be completed. Please set your dataset with more than 1 location." -message2<-"" -locations <- unique(pheno$locationDbId) -acc <- unique(pheno$germplasmDbId) -subGen <- unique(subset(pheno, select=c(germplasmDbId, germplasmName))) - - -if (! length(locations)>1){ - - png(AMMIFile, height = 130, width=800) - z<-tableGrob(message) - grid.arrange(z) - dev.off() - - sub1 <- unique(subset(pheno, locationDbId == locations[1], select=c(locationDbId, locationName))) - png(figure1_file_name, height = 100, width=800) - p<-tableGrob(sub1) - grid.arrange(p) - dev.off() - - acc <- unique(pheno$germplasmDbId) - png(figure2_file_name, height = (21*length(acc)), width = 800) - sub2 <- unique(subset(pheno, locationDbId == locations[1], select=c(germplasmDbId, germplasmName))) - q<-tableGrob(sub2) - grid.arrange(q) - dev.off() -} else { - cat("Starting stability analysis...","\n") +imputation_accuracy <- function(checkDF, fMissing){ + checkDF$Gen <- as.character(checkDF$Gen) + checkDF$Env <- as.character(checkDF$Env) + + realData <- pheno_imputation(checkDF) + + # Subseting to check imputation accuracy + realData <- realData[order(realData$Gen, realData$Env), ] + data_missing <- sample(rownames(realData), nrow(realData)*fMissing) + testindDF <- realData + testindDF <- testindDF[order(testindDF$Gen, testindDF$Env), ] + testindDF[rownames(testindDF)%in% data_missing, "Yield"] <- NA + + testindDF$Gen <- as.factor(testindDF$Gen) + testindDF$Env <- as.factor(testindDF$Env) + testindDF$Rep <- as.factor(testindDF$Rep) + + imputed_test <- pheno_imputation(testindDF) + imputed_test <- imputed_test[order(imputed_test$Gen, imputed_test$Env), ] + cat("Imput Acc ok!\n") + corrImput <- cor(realData$Yield,imputed_test$Yield, use = "complete") + return(corrImput) + } -cat("create model", "\n") -summary(pheno) -summary(env) -summary(gen) -summary(rep) -model<- with(pheno,AMMI(env, gen, rep, pheno[,col], console=FALSE)) - -cat("marker 1", "\n") -anova <-format(round(model$ANOVA, 3)) -cat("marker 2", "\n") -analysis <- model$analysis -cat("marker 3", "\n") -anova -cat("marker 4", "\n") -analysis - -cat("DONE WITH MODEL", "\n") - -png(AMMIFile, height=130, width=800) -p<-tableGrob(anova) -grid.arrange(p) -dev.off() - - -if(method=="ammi"){ - - cat("running ammi", "\n") - # Biplot and Triplot - png(figure1_file_name,height=400) - plot(model, first=0,second=1, number=TRUE, xlab = study_trait) - dev.off() - - tam <- nrow(model$analysis) - if (tam >2){ - png(figure2_file_name, height= 300) - plot(model, type=2, number=TRUE, xlab = study_trait) - dev.off() - }else{ - png(figure2_file_name, height=5, width=5) - y <- tableGrob(message2) - grid.arrange(y) - dev.off() + + +# Setting dataframe to required format +pheno <- pheno[,colnames(pheno) %in% c("locationName", "germplasmName", "replicate", "entryType", study_trait[1])] + + + +######################################## +# Quality Control # +# Checking conditions to run analysis # +# # +######################################## + +# Getting locations, accessions and reps +accessions_sel <- unique(pheno$germplasmName) +# Some breeding programs are using accession named Filler +accessions_sel <- accessions_sel[!accessions_sel == "Filler" | !accessions_sel == "filler"] + +# Taking test accessions +dat <- pheno %>% dplyr::select(germplasmName, locationName, replicate, entryType, trait=study_trait[1]) +dat <- dat[dat$entryType == "test",] +nLoc <- length(unique(dat$locationName)) +nReps <- max(dat$replicate) + +# Number of reps mat be different for checks +# Fixing here +pheno <- pheno[pheno$replicate <= nReps & pheno$germplasmName %in% accessions_sel,] + +# Preparing dataset for for analysis +dat1 <- pheno %>% dplyr::select(germplasmName, locationName, replicate, trait=study_trait[1]) + +fMissing <- nrow(dat1[is.na(dat1$trait),])/nrow(dat1) +message2 <- paste0("The frequency of missing data is too high. Please, check the dataset.") + +summaryData <- data.frame(mean = mean(dat1[,4], na.rm=TRUE), + min = min(dat1[,4], na.rm=TRUE), + maxV = max(dat1[,4], na.rm=TRUE), + sdV = sd(dat1[,4], na.rm=TRUE), + missing = fMissing + ) + +myJson <- jsonlite::toJSON(summaryData) +jsonlite::write_json(myJson, jsonSummary) + + +find_duplications <- function(inData){ + # Step 1: Identify duplicate rows based on the combination of four columns + duplicated_rows <- duplicated(inData[, c("Gen", "Env", "Rep")]) + + # Step 2: Calculate the average of 'Yield' for each unique combination of the four columns + averages <- aggregate(Yield ~ Gen + Env + Rep, data = dat1, FUN = mean) + + # Step 3: Replace the duplicated rows with the calculated average + pheno_unique <- inData[!duplicated_rows, ] # Keep only unique rows + pheno_unique <- merge(pheno_unique, averages, by = c("Gen", "Env", "Rep"), all.x = TRUE) + + # If there are any missing values (for rows that were not duplicated), fill them with the original 'Yield' + pheno_unique$Yield <- ifelse(is.na(pheno_unique$Yield.y), pheno_unique$Yield.x, pheno_unique$Yield.y) + + # Remove unnecessary columns (if needed) + pheno_unique <- pheno_unique[, !(names(pheno_unique) %in% c("Yield.x", "Yield.y"))] + + return(pheno_unique) + + # View the resulting data frame +} + + +dat1$germplasmName <- as.factor(dat1$germplasmName) +dat1$locationName <- as.factor(dat1$locationName) +dat1$replicate <- as.factor(dat1$replicate) +dat1$trait <- as.double(dat1$trait) + +# Running model from stability package +# Assuming dat1 is your data frame +dat1 <- dat1[order(dat1$germplasmName, dat1$locationName), ] +colnames(dat1) <- c("Gen", "Env", "Rep", "Yield") + + +dat2 <- find_duplications(dat1) + +if(imputPheno == "imput_yes"){ + testCor <- imputation_accuracy(dat2, fMissing) + dat2 <- pheno_imputation(dat2) + cat("The imputation accuracy is: ", sprintf("%.3f",testCor), "\n") +} + +# testing if all accessions have the same number of reps in all locations +dat <- na.omit(dat2) +replicate_df <- data.frame(replicate = tapply(dat$Yield, list(dat$Gen, dat$Env), length)) +replicate_df$germplasmName <- rownames(replicate_df) +gathered_df <- gather(replicate_df, key = "replicate_locations", value = "value", -germplasmName) +repAcc <- unique(gathered_df$value) +message1 <- paste0("Please, check dataset for accessions per locations and reps.") + +# Saving error message +errorMessages <- c() +if(fMissing >= 0.4){ + errorMessages <- append(errorMessages, message2) +}else if(length(repAcc)>1){ + errorMessages <- append(errorMessages, message1) +} + +# Setting second rep to compare augmented design with 1 rep per location +if(nReps == 1 && stability_method == "ammi"){ + dat2.1 <- dat2 + dat2.1$Rep <- as.numeric(dat2.1$Rep) + dat2.1$Rep <- 2 + dat2$Rep <- as.numeric(dat2$Rep) + dat2 <- rbind(dat2,dat2.1) + dat2$Rep <- as.factor(dat2$Rep) +}else if (nReps == 1 && stability_method == "gge"){ + errorMessages <- append(errorMessages, "The number of replication must be greater than 1 for gge.") +} + + +if(stability_method=="ammi" && length(errorMessages) == 0 ){ + library("agricolae") + + model <- NULL + # Running AMMI model + tryCatch({ + # Attempt to execute the code + model<- with(dat2,AMMI(Env, Gen, Rep, Yield, console=FALSE)) + }, error = function(e) { + # Handle the error by printing a message + cat("An error occurred:", conditionMessage(e), "\n") + append(errorMessages, paste0("An error occurred:", conditionMessage(e))) + }) + + index<-index.AMMI(model) + index <- tibble::rownames_to_column(index, "Accession") + indexDF <- data.frame(Accession = index$Accession, ASV = index$ASV, Rank = index$rASV) + + # Adding a column to scale ASV + # This column will be used to plot stability lines + slopes <- data.frame(Accession = index$Accession, slope = index$ASV, scaled= "NA") + slopes$scaled <-as.vector(scale(slopes$slope, center = T, scale = F)) + indexDF <- left_join(indexDF, slopes, by="Accession") + + indexDF <- as.data.frame(indexDF) + indexDF <- indexDF[order(indexDF[,3]),] + + ## Preparing table with GxE effects + list_data <- list(model$genXenv) + dataGraphic <- data.frame(plyr::ldply(list_data)) + rownames(dataGraphic) <- model$means$GEN[1:(nrow(dataGraphic))] + dataGraphic <- tibble::rownames_to_column(dataGraphic, "Accession") + + myMeans <- data.frame(Accession = model$means$GEN, location = model$means$ENV, means = model$means$Yield) + myMeans$average <- myMeans$means + myMeans$means <- as.vector(scale(myMeans$means, center = T, scale = FALSE)) + meanDF <- data.frame(tapply(myMeans$means, myMeans$Accession, mean)) + meanDF$Accession <- rownames(meanDF) + colnames(meanDF) <- c("means", "Accession") + meanDF <- left_join(meanDF, slopes, by = "Accession") %>% select(Accession, means, slope) + meanDF$start <- 0 + meanDF$end <- length(unique(pheno$locationName)) + minDF <- data.frame(tapply(myMeans$means, myMeans$Accession, min)) + minDF$Accession <- rownames(minDF) + colnames(minDF) <- c("minimum", "Accession") + + meanDFF <- left_join(meanDF, minDF, by = "Accession") + meanDFF$value <- meanDFF$minimum*meanDF$slope + meanDFF <- gather(meanDFF, key = "T", value = "X",start,end) %>% select(Accession, minimum, value, X) %>% arrange(Accession) + meanDFF <- left_join(meanDFF, slopes, by = "Accession") + + for ( i in 1:nrow(meanDFF)){ + if(meanDFF$X[i] == 0){ + meanDFF$value[i] <- 0 + }else{ + meanDFF$value[i] <- (-1)*meanDFF$slope[i] + } } - }else if( method=="gge"){ - dat1 <- pheno %>% select(germplasmName, locationName, trait=study_trait) - head(dat1) + scaled_values <- (meanDFF$value - min(meanDFF$value)) / (max(meanDFF$value) - min(meanDFF$value)) + meanDFF$value <- scaled_values - model1 <- gge(dat1, trait~germplasmName*locationName, scale=FALSE) - - name1 = paste(study_trait,"- GGE Biplot", sep=" ") + myData <- data.frame(pivot_longer(dataGraphic, 2:ncol(dataGraphic), names_to = "location", values_to = "Effect")) + myData$location <- gsub("\\."," ", myData$location) + + preFinal <- left_join(myData, myMeans, by = c("Accession", "location")) + selLocations <- data.frame(locName = unique(preFinal$location), locNumber = c(1:length(unique(preFinal$location)))) + preFinal <- left_join(preFinal, selLocations, by = c("location"="locName")) + + finalDF <- left_join(preFinal, indexDF, by="Accession") %>% dplyr::select(Accession, location, Effect, Rank, means, slope, scaled) + finalDF <- finalDF[order(finalDF$Rank, -finalDF$Effect),] + finalDF$Effect <- sprintf("%.3f", finalDF$Effect) + finalDF$means <- sprintf("%.3f", finalDF$means) + finalDF$slope <- sprintf("%.3f", finalDF$slope) + finalDF$scaled <- sprintf("%.3f", finalDF$scaled) + + graphicJson <- jsonlite::toJSON(meanDFF) + jsonlite::write_json(graphicJson, graphFile) + + # Parsing files to combine Accession, locations, effects, rank, averages, slopes + preFinal <- left_join(myData, myMeans, by = c("Accession", "location")) + finalDF <- left_join(preFinal, indexDF, by="Accession") %>% dplyr::select(Accession, location, Effect, Rank, average, slope, scaled) + colnames(finalDF)[5] <- "means" + if(imputPheno == "imput_yes"){finalDF$imputAcc <- testCor} + finalDF <- finalDF[order(finalDF$Rank, -finalDF$Effect),] + + myJson <- jsonlite::toJSON(finalDF) + jsonlite::write_json(myJson, jsonFile) + + }else if( stability_method=="gge" && length(errorMessages) == 0){ + library("stability") + cat("running gge", "\n") + # dat2 <- pheno_imputation(dat2) + + dat2$Env <- as.character(dat2$Env) + effectsDF <- NULL + # Use tryCatch to handle potential errors + tryCatch({ + # Attempt to execute the code + effectsDF <- stability::ge_means(.data = dat2, .y = Yield, .gen = Gen, .env = Env) + }, error = function(e) { + # Handle the error by printing a message + cat("An error occurred:", conditionMessage(e), "\n") + append(errorMessages, paste0("An error occurred:", conditionMessage(e))) + }) - png(figure1_file_name, height=400, width=500) - biplot(model1, main=name1, flip=c(1,0), origin=0, hull=TRUE) - dev.off() - - model2 <- gge(dat1, trait~germplasmName*locationName, scale=TRUE) - png(figure2_file_name,height=400, width=500) - biplot(model2, main=name1, flip=c(1,1), origin=0) - dev.off() + effectsDF <- stability::ge_means(.data=dat2, .y= Yield, .gen=Gen, .env=Env) + + test <- data.frame(effectsDF$ge_ranks) + test$location <- rownames(test) + gathered_df <- gather(test, key = "rank", value = "genotypes", -location) + gathered_df$rank <- gsub("X", "", gathered_df$rank) + gathered_df$rank <- as.numeric(gathered_df$rank) + + test1 <- data.frame(effectsDF$ge_means) + gathered_mean <- gather(test1, key = "locations", value = "means", -Gen) + + + gathered_df$track <- paste0(gathered_df$genotypes,"_",gathered_df$location) + gathered_mean$track <- paste0(gathered_mean$Gen,"_",gathered_mean$locations) + + gathered_df$track <- gsub(" ","",gathered_df$track) + gathered_df$track <- gsub("\\.","",gathered_df$track) + + gathered_mean$track <- gsub(" ","",gathered_mean$track) + gathered_mean$track <- gsub("\\.","",gathered_mean$track) + + + finalDF <- left_join(gathered_df, gathered_mean, by = "track") %>% dplyr::select( + genotypes, locations, means, rank + ) + finalDF$locations <- gsub("\\."," ",finalDF$locations) + + rankDF <- data.frame(sumRank=tapply(finalDF$rank, finalDF$genotypes, sum)) + rankDF$genotypes <- rownames(rankDF) + rankDF$genotypesRank <- rank(rankDF$sumRank, ties.method = "min") + finalDF <- left_join(finalDF, rankDF, by = "genotypes") %>% dplyr::select(genotypes, locations, means, rank, genotypesRank) + colnames(finalDF) <- c("genotypes","locations", "means", "locationRank","genotypeRank") + finalDF$means <- sprintf("%.2f", finalDF$means) + + colnames(finalDF) <- c("Accession", "location", "means", "locationRank", "genotypeRank") + if(imputPheno == "imput_yes"){finalDF$imputAcc <- testCor} + # Saving Json + myJson <- jsonlite::toJSON(finalDF) + jsonlite::write_json(myJson, jsonFile) + + + ## Preparing Graphic dataset + prepGraph <- as.data.frame(tapply(dat2$Yield, list(dat2$Gen, dat2$Env), mean)) + prepGraph <- tibble::rownames_to_column(prepGraph, var="Accession") + meanGraph <- gather(prepGraph, key = "location", value = "mean", -Accession) + prepGraph2 <- as.data.frame(tapply(dat2$Yield, list(dat2$Gen, dat2$Env), sd)) + prepGraph2 <- tibble::rownames_to_column(prepGraph2, var="Accession") + sdGraph <- gather(prepGraph2, key = "location", value = "sd", -Accession) + finalGraph <- left_join(meanGraph, sdGraph, by=c("Accession","location")) + + graphicJson <- jsonlite::toJSON(finalGraph) + jsonlite::write_json(graphicJson, graphFile) + } + +if ( is.null(errorMessages) == F ) { + print(sprintf("Writing Error Messages to file: %s", messageFile)) + print(errorMessages) + write(errorMessages, messageFile) +} + + \ No newline at end of file diff --git a/R/unreplicated_diagonal_design.R b/R/unreplicated_diagonal_design.R new file mode 100644 index 0000000000..e69de29bb2 diff --git a/R/urdd_design.R b/R/urdd_design.R new file mode 100644 index 0000000000..f58f26882b --- /dev/null +++ b/R/urdd_design.R @@ -0,0 +1,94 @@ +## Script for unreplicated diagonal designs + +args <- commandArgs(TRUE) + +if (length(args) == 0) { + message("No arguments supplied.") + paramfile <- "" +} else { + for (i in seq_along(args)) { + message(paste("Processing arg", args[[i]])) + eval(parse(text = args[[i]])) + } +} + +## Source Files +source(paramfile) +basefile <- tools::file_path_sans_ext(paramfile) + +## Entry stocks +n_stocks <- length(stocks) +n_checks <- length(controls) + +## Separating treatments from checks +treatments <- stocks[!stocks %in% controls] + +## Setting the same number of treatments per block +if (is.null(n_blocks)) { + n_ind <- n_lines + n_ind_block <- n_lines +} else { + n_ind <- n_lines / n_blocks + n_ind_block <- rep(n_ind, n_blocks) +} + +n_controls <- length(controls) + +type_design <- if (n_blocks > 1) "DBUDC" else "SUDC" +layout_type <- if (layout == "serpentine") "serpentine" else "cartesian" + +treatment_list <- data.frame( + entry = seq_len(n_stocks), + name = c(controls, treatments) +) + +start_plot <- switch( + as.character(serie), + "1" = 1, + "2" = 101, + "3" = 1001, + 1 # default to 1 if none matched +) + +## Grant the right format +n_row <- as.integer(n_row) +n_col <- as.integer(n_col) +n_lines <- as.integer(n_lines) +n_checks <- as.integer(n_checks) + + +## FieldHub design +output <- capture.output({ + multi_diag <- FielDHub::diagonal_arrangement( + nrows = n_row, + ncols = n_col, + lines = n_lines, + planter = layout_type, + plotNumber = start_plot, + kindExpt = type_design, + blocks = n_ind_block, + checks = n_checks, + l = 1, + data = treatment_list + ) +}) + +if (any(grepl("Field dimensions do not fit", output))) { + error_file <- paste0(basefile, ".design.error") + writeLines(output, con = error_file) + quit(status = 1) +} + +field_book <- multi_diag$fieldBook + +## Fixing names to match with Breedbase +field_book$EXPT <- gsub("Block", "", field_book$EXPT) +field_book$CHECKS[field_book$CHECKS > 0] <- 1 + +print(head(field_book, 10)) + +## Save result files +out_file <- paste0(basefile, ".design") +sink(out_file) +write.table(field_book, quote = FALSE, sep = "\t", row.names = FALSE) +sink() diff --git a/README.md b/README.md index 8aa14f85bb..ca6e28f7c7 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ This repo contains code behind the Breedbase sys Docker images are available for the system at the Breedbase DockerHub page, so you shouldn't need to build the system yourself. Build scripts are available in the breedbase_dockerfile GitHub repo. +To run Breedbase, the easiest way is to git clone the repo https://github.com/solgenomics/breedbase_site/. Required prerequisites on the system are git, nginx, docker and docker compose. Follow the instructions on the breedbase_site wiki. + Contact SGN using the contact form on the page https://solgenomics.net/contact/form for more information. Users should refer to the user's manual. diff --git a/bin/Reports/phenotype_properties_check.pl b/bin/Reports/phenotype_properties_check.pl new file mode 100644 index 0000000000..e637a5e75a --- /dev/null +++ b/bin/Reports/phenotype_properties_check.pl @@ -0,0 +1,256 @@ +#!/usr/bin/env perl + +=head1 + +phenotype_properties_check.pl + +=head1 SYNOPSIS + + \$phenotype_properties_check.pl -U [dbuser] -P [dbpass] -H [dbhost] -D [dbname] + +=head1 COMMAND-LINE OPTIONS + -U db username (postgres) + -P db password + -H host name + -D database name + -o output directory + -f output filename + -s start date (YYYY-MM-DD) + -e end date (YYYY-MM-DD) + +=head2 DESCRIPTION + +Prints phenotype QC issues as JSON. + +=head2 AUTHOR + +chris simoes (ccs263@cornell.edu) + +March 2025 + +=cut + +use strict; +use warnings; +use Getopt::Std; +use DBI; +use JSON; +use Bio::Chado::Schema; +use CXGN::Metadata::Schema; +use CXGN::Phenome::Schema; +use CXGN::Trial; +use SGN::Model::Cvterm; +use Excel::Writer::XLSX; + +our %opt; +getopts('H:D:U:P:o:f:s:e:', \%opt); + +my $dbhost = $opt{H} // 'localhost'; +my $dbname = $opt{D} // 'breedbase'; +my $dbuser = $opt{U} // 'postgres'; +my $dbpass = $opt{P} // ''; +my $out_directory = $opt{o} // ''; +my $filename = $opt{f} // ''; +my $start_date = $opt{s} // ''; +my $end_date = $opt{e} // ''; + +warn "Parsed start_date: [$start_date]"; +warn "Parsed end_date: [$end_date]"; + +print STDERR "Connecting to database $dbname on $dbhost as $dbuser...\n"; + +my $dbh = DBI->connect( + "dbi:Pg:dbname=$dbname;host=$dbhost", + $dbuser, + $dbpass, + { AutoCommit => 0, RaiseError => 1 } +); + +my $schema = Bio::Chado::Schema->connect(sub { $dbh }); + +my $rs = $schema->resultset("Project::Project")->search( + { + create_date => { + -between => [ "$start_date 00:00:00", "$end_date 23:59:59" ] + } + }, + { + select => ['project_id', 'create_date'], + as => ['project_id', 'create_date'] + } +); + +my @trial_ids; +my %trial_create_dates; + +while (my $row = $rs->next) { + my $trial_id = $row->get_column('project_id'); + my $create_date = $row->get_column('create_date'); + $create_date =~ s/\s.*$// if $create_date; + + push @trial_ids, $trial_id; + $trial_create_dates{$trial_id} = $create_date; +} + +my (%trial_data, %trial_cols); +foreach my $trial_id (@trial_ids) { + my $t; + eval { $t = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id }); }; + next if $@ || !defined $t; + + my $trial_name = $t->get_name; + my $location_name = $t->get_location ? $t->get_location()->[1] : "Unknown"; + my $breeding_program_name = $t->get_breeding_program(); + my $year = $t->get_year(); + my $test_type = $t->get_design_type(); + + next unless defined $year; + next if !defined($test_type) || $test_type eq "Analise"; + + $trial_cols{$trial_id} = [ $year, $breeding_program_name, $trial_name, $location_name ]; + + my $traits = $t->get_traits_assayed(); + my @trait_names = map { $_->[0] } @$traits; + + print("Running project $trial_name \n"); + + my $sth = $dbh->prepare(qq{ + SELECT + acc.uniquename AS accession_name, + plot.uniquename AS plot_name, + trait.name AS trait_name, + ph.value AS observed_value + FROM nd_experiment e + JOIN nd_experiment_project ep ON (e.nd_experiment_id = ep.nd_experiment_id) + JOIN nd_experiment_phenotype nep ON (nep.nd_experiment_id = ep.nd_experiment_id) + JOIN phenotype ph ON (nep.phenotype_id = ph.phenotype_id) + JOIN cvterm trait ON (ph.cvalue_id = trait.cvterm_id) + JOIN nd_experiment_stock nes ON (e.nd_experiment_id = nes.nd_experiment_id) + JOIN stock plot ON (nes.stock_id = plot.stock_id) + LEFT JOIN stock_relationship rel ON (plot.stock_id = rel.object_id) + LEFT JOIN stock acc ON (rel.subject_id = acc.stock_id) + WHERE ep.project_id = ? + }); + $sth->execute($trial_id); + + my @data; + while (my ($accession, $plot, $trait, $value) = $sth->fetchrow_array) { + $accession //= 'Unknown'; + push @data, [ $accession, $plot, $trait, $value ]; + } + + $trial_data{$trial_id} = \@data; +} + + +my $minimum_cvterm_id = $schema->resultset("Cv::Cvterm")->find({ name => 'trait_minimum' })->cvterm_id; +my $maximum_cvterm_id = $schema->resultset("Cv::Cvterm")->find({ name => 'trait_maximum' })->cvterm_id; + +my @trial_ids = keys %trial_data; +my $total = scalar(@trial_ids); +my $counter = 0; + +my @qc_issues; +foreach my $trial_id (@trial_ids) { + + $counter++; + print("Getting data from trial_id=$trial_id counter: $counter/$total\n"); + + my ($year, $breeding_program_name, $trial_name, $location_name) = @{$trial_cols{$trial_id}}; + my $create_date = $trial_create_dates{$trial_id} || 'NULL'; + + foreach my $row (@{$trial_data{$trial_id}}) { + my ($accession, $plot, $trait_name, $observed_value) = @$row; + + my $trait_cvterm = $schema->resultset("Cv::Cvterm")->find({ name => $trait_name }); + next unless $trait_cvterm; + my $trait_id = $trait_cvterm->cvterm_id; + + my $min_value_rs = $schema->resultset("Cv::Cvtermprop")->search({ + 'cvterm_id' => $trait_id, + 'type_id' => $minimum_cvterm_id + }); + my $min_value = $min_value_rs->first ? $min_value_rs->first->value : undef; + + my $max_value_rs = $schema->resultset("Cv::Cvtermprop")->search({ + 'cvterm_id' => $trait_id, + 'type_id' => $maximum_cvterm_id + }); + my $max_value = $max_value_rs->first ? $max_value_rs->first->value : undef; + + next unless defined $min_value && defined $max_value; + next if $min_value eq $max_value; + + my $qc_type; + + if (defined $observed_value && $observed_value !~ /^-?\d+(\.\d+)?$/) { + $qc_type = "invalid numeric format"; + } + elsif (defined $observed_value && $observed_value < $min_value) { + $qc_type = "below minimum"; + } + elsif (defined $observed_value && $observed_value > $max_value) { + $qc_type = "above maximum"; + } + + if ($qc_type) { + push @qc_issues, { + year => $year, + breeding_program_name => $breeding_program_name, + trial_name => $trial_name, + trial_location => $location_name, + create_date => $create_date, + accession => $accession, + plot => $plot, + trait_name => $trait_name, + observed => $observed_value, + minimum => $min_value, + maximum => $max_value, + type => $qc_type + }; + } + } +} + +# Create an Excel workbook +my $excel_file_path = $out_directory . $filename . '.xlsx'; + +my $workbook = Excel::Writer::XLSX->new($excel_file_path) + or die "Could not create Excel file $excel_file_path: $!"; + +my $worksheet = $workbook->add_worksheet('Phenotype QC'); + +# Write header row +my @columns = qw( + year + breeding_program_name + trial_name + trial_location + create_date + accession + plot + trait_name + observed + minimum + maximum + type +); +my $col_index = 0; +for my $col_name (@columns) { + $worksheet->write(0, $col_index++, $col_name); +} + +# Write data rows +my $row_index = 1; +for my $record (@qc_issues) { + $col_index = 0; + for my $col_name (@columns) { + $worksheet->write($row_index, $col_index++, $record->{$col_name}); + } + $row_index++; +} + +print "Excel file saved in $excel_file_path\n\n"; + +$dbh->disconnect(); +exit(0); \ No newline at end of file diff --git a/bin/bdb_update_blast_dbs.pl b/bin/bdb_update_blast_dbs.pl index 6368d9bacd..8a32b98104 100755 --- a/bin/bdb_update_blast_dbs.pl +++ b/bin/bdb_update_blast_dbs.pl @@ -11,7 +11,6 @@ use File::Spec; use File::Temp qw/tempfile/; -use CXGN::Tools::Wget qw/wget_filter/; use SGN::Schema; use CXGN::Blast; use CXGN::DB::InsertDBH; @@ -20,20 +19,20 @@ sub usage { my $sgn_schema = shift; my $message = shift || ''; $message = "Error: $message\n" if $message; - + # my $file_bases = join '', sort map ' '.$_->file_base."\n", CXGN::BlastDB->retrieve_all($sgn_schema, $opt{d}); - + die < - + Go over all the BLAST databases we keep in stock and update them if needed. When run with just the -g option, goes over all the BLAST dbs listed in the sgn.blast_db table and updates them if needed, putting them under the top-level BLAST db path given with the -d option. - + Options: -H @@ -52,6 +51,8 @@ sub usage { -f force-update the DB with the given file base (e.g. 'genbank/nr') + -a force update all dbs in blast dir - override needs_update and run makeblastdb on all blast datasets found in filebase + Current list of file_bases: EOU @@ -59,7 +60,7 @@ sub usage { our %opt; -getopts('xt:d:f:H:D:p:U:h',\%opt) or die "Invalid arguments"; +getopts('axt:d:f:H:D:p:U:h',\%opt) or die "Invalid arguments"; $opt{t} ||= File::Spec->tmpdir; @@ -67,10 +68,10 @@ sub usage { my $dbh; -if (!$opt{p}) { +if (!$opt{p}) { $dbh = CXGN::DB::InsertDBH->new( { dbhost => $opt{H}, dbname => $opt{D} }); } -else { +else { $dbh = CXGN::DB::Connection->new( { dbhost => $opt{H}, dbname => $opt{D}, dbpass => $opt{p}, dbuser => $opt{U} }); } @@ -86,37 +87,46 @@ sub usage { my $bdbs = CXGN::Blast->new( sgn_schema => $sgn_schema, dbpath => $opt{d} ); -my @dbs = $opt{f} ? CXGN::Blast->search( $sgn_schema, $opt{d}, file_base => $opt{f} ) +my @dbs = $opt{f} ? CXGN::Blast->search( $sgn_schema, $opt{d}, file_base => $opt{f} ) : CXGN::Blast->retrieve_all($sgn_schema, $opt{d}); unless(@dbs) { print $opt{f} ? "No database found with file_base='$opt{f}'.\n" : "No dbs found in database.\n"; } +my $count; +my @errs; + foreach my $db (@dbs) { - - print STDERR "Processing database ".$db->title()."\n"; - + + print STDERR "Processing database ".$db->title(). "\n". $db->file_base . "\n" ; #check if the blast db needs an update - unless($opt{f} || $db->needs_update) { + unless($opt{f} || $db->needs_update || $opt{a}) { print $db->file_base." is up to date.\n"; next; } - + print STDERR "checking source url..\n"; + ##Not usig source_url anymore. Source files need to be in the blast db dir #skip the DB if it does not have a source url defined - unless($db->source_url) { - warn $db->file_base." needs to be updated, but has no source_url. Skipped.\n"; - next; - } - + #unless($db->source_url) { + # warn $db->file_base." needs to be updated, but has no source_url. Skipped.\n"; + # next; + #} + ########### + ###do not use source_url. Need to make sure all db fasta files are in the blast basedir + #my $source_url = $db->source_url ; + #$source_url =~ s/^ftp:\/\/ftp.sgn.cornell.edu/http:\/\/solgenomics.net\/ftp/; + ########### + my $file_path = $opt{d} . $db->file_base; + if( $opt{x} ) { - print "Would update ".$db->file_base." from source url ".$db->source_url."\n"; - next; + print "Would update ".$db->file_base." from file ".$file_path."\n"; + next; } else { - print "Updating ".$db->file_base." from source url...\n"; + print "Updating ".$db->file_base." from file...\n"; } - + eval { print STDERR "Checking permissions...\n"; @@ -125,31 +135,37 @@ sub usage { if( my $perm_error = $db->check_format_permissions() ) { die "Cannot format ".$db->file_base.":\n$perm_error"; } - + #download the sequences from the source url to a tempfile - print STDERR "Downloading source (".$db->source_url.")...\n"; - my (undef,$sourcefile) = tempfile('blastdb-source-XXXXXXXX', - DIR => $opt{t}, - UNLINK => 1, - ); - - my $wget_opts = { cache => 0 }; - $wget_opts->{gunzip} = 1 if $db->source_url =~ /\.gz$/i; - wget_filter( $db->source_url => $sourcefile, $wget_opts ); - + print STDERR "Reading source file (".$file_path.")...\n"; + + #### no longer used. Check if some blast dbs need to be copied manually to the blast basedir + #use source_url only if file not found in file_path + #my (undef,$sourcefile) = tempfile('blastdb-source-XXXXXXXX', + # DIR => $opt{t}, + # UNLINK => 1, + # ); + +# my $wget_opts = { cache => 0 }; +# $wget_opts->{gunzip} = 1 if $source_url =~ /\.gz$/i; +# wget_filter( $source_url => $sourcefile, $wget_opts ); + #formatdb it into the correct place print STDERR "Formatting database..."; - $db->format_from_file($sourcefile); - - unlink $sourcefile or warn "$! unlinking tempfile '$sourcefile'"; - + $db->format_from_file($file_path); + + #unlink $sourcefile or warn "$! unlinking tempfile '$sourcefile'"; + print $db->file_base." done.\n"; }; if( $EVAL_ERROR ) { - print "Update failed for ".$db->file_base.":\n$EVAL_ERROR"; - } - -} + print STDERR "Update failed for ".$db->file_base.":\n$EVAL_ERROR"; + push(@errs , "Update failed for ".$db->file_base.":\n$EVAL_ERROR\n"); + } + $count++; + } -$dbh->disconnect(); + print STDERR "Updated $count blast dbs\n"; + print STDERR join(", ", @errs); +$dbh->disconnect(); diff --git a/bin/convert_ncsu_excel_to_obo.pl b/bin/convert_ncsu_excel_to_obo.pl new file mode 100644 index 0000000000..c1b1008df2 --- /dev/null +++ b/bin/convert_ncsu_excel_to_obo.pl @@ -0,0 +1,339 @@ + +=head1 NAME + +convert_excel_to_obo - a script to convert a spreadsheet based representation of an ontology to an obo file format + +=head1 DESCRIPTION + +Based on CXGN::File::Parse, this script can parse tab delimited or Excel formats (xls or xlsx) as follows: + +perl convert_excel_to_obo.pl -n CO_999 -i file.xlsx -o ontology.obo + +=head1 AUTHOR + +Lukas Mueller + +October 2024 + +=cut + +use strict; + +use utf8; +use Getopt::Std; +use Data::Dumper; +use CXGN::File::Parse; + +our ($opt_n, $opt_i, $opt_o, $opt_h); + +getopts('n:i:o:h'); + +my $file = $opt_i; +my $ontology_name = $opt_n || "GENERIC"; + +if (!$file) { + die "Please privde a file using the -i parameter."; +} + +print STDERR "ONTOLOGY NAME: $ontology_name\n"; + +my $outfile = $file.".obo"; +my $cvpropfile = $file.".props"; + +open(my $F, ">", $outfile) || die "Can't open file $outfile\n"; +open(my $G, ">", $cvpropfile) || die "Can't open cvprop file $cvpropfile for writing"; + +#Curation Variable ID Variable name Variable label Variable description Variable synonyms Context of use Growth stage Variable status Variable Xref Institution Scientist Date Language Crop Trait ID Entity Attribute Trait name Trait class Trait description Trait synonyms Main trait abbreviation Alternative trait abbreviations Trait status Trait Xref Method ID Method name Method class Method description Method Abbreviation Formula Method reference Scale ID Scale name Scale Abbreviation Scale class Scale Xref Cat 1 code Cat 1 description Cat 2 code Cat 2 description Cat 3 code Cat 3 description Cat 4 code Cat 4 description Cat 5 code Cat 5 description Cat 6 code Cat 6 description Cat 7 code Cat 7 description Cat 8 code Cat 8 description Cat 9 code Cat 9 description Cat 10 code Cat 10 description + +my @col_headers = ("Variable"," Term Name - BB", "Trait class", "Term Definition", "Variable Full Name", "Synonyms", "Trait - CO", "Main trait abbreviation", "Entity", "Attribute", "Method Name", "Method class", "Method description", "Method Abbreviation", "Formula", "Scale name", "Scale abbreviation", "Scale class", "Category 1", "Category 2", "Category 3", "Category 4", "Category 5", "Category 6", "Category 7", "Category 8", "Category 9", "Category 10", "Category 11", "Category 12" ); + +# column labels +# +my $trait_class = "Trait class"; +my $trait_name = "Trait name"; +my $trait_definition = "Trait description"; +my $trait_synonyms = "Trait synonyms"; +my $variable_synonyms = "Variable synonyms"; +my $trait_id = "Trait ID"; +my $variable_name = "Variable name"; +my $variable_definition = "Variable description"; +my $variable_label = "Variable label"; +my $variable_id = "Variable ID"; +my $method_id = "Method ID"; +my $method_name = "Method name"; +my $method_class = "Method class"; +my $method_description = "Method description"; +my $scale_id = "Scale ID"; +my $scale_name = "Scale name"; +my $scale_class = "Scale class"; +my $scale_description = "Scale description"; +my $scale_abbreviation = "Scale abbreviation"; +my $entity = "Entity"; +my $attribute = "Attribute"; +my $categories = "Categories"; +my $class_id = "Class ID"; +my $class_name = "Class name"; + +my $parser = CXGN::File::Parse->new( file => $file ); + +my $parsed = $parser->parse(); + +if ($parsed->{errors}) { + warn "The following errors occurred while parsing file $file: ".Dumper($parsed->{errors})."\n"; +} + +my $data = $parsed->{data}; + +# get all the trait classes +# +my %trait_classes; +my %traits; +my %variables; + +foreach my $d (@$data) { + $trait_classes{$d->{$trait_class}}->{count}++; +} +print STDERR "TRAIT CLASSES: ".Dumper(\%trait_classes); + +foreach my $d (@$data) { + my $tn = $d->{$trait_name}; + print STDERR "Parsing TRAIT NAME $trait_name\n"; + if (! $tn) { next; } + $traits{$tn}->{$trait_id} = $d->{$trait_id}; + $traits{$tn}->{$trait_class} = $d->{$trait_class}; + + print STDERR "TRAIT NAME $trait_name has TRAIT CLASS $d->{$trait_class}\n"; + + $traits{$tn}->{$trait_definition} = $d->{$trait_definition}; +} + +print STDERR "TRAITS: ".Dumper(\%traits); + + + +foreach my $d (@$data) { + my $vn = $d->{$variable_name}; + if (! $vn) { next; } + $variables{$vn}->{$variable_id} = $d->{$variable_id}; + $variables{$vn}->{$variable_synonyms} = $d->{$variable_synonyms}; + $variables{$vn}->{$trait_name} = $d->{$trait_name}; + $variables{$vn}->{$trait_definition} = $d->{$trait_definition}; + $variables{$vn}->{$entity} = $d->{$entity}; + $variables{$vn}->{$attribute} = $d->{$attribute}; + $variables{$vn}->{$method_name} = $d->{$method_name}; + $variables{$vn}->{$scale_abbreviation} = $d->{$scale_abbreviation}; + $variables{$vn}->{$variable_label} = $d->{$variable_label}; + $variables{$vn}->{$scale_name} = $d->{$scale_name}; + $variables{$vn}->{$scale_class} = $d->{$scale_class}; + $variables{$vn}->{$categories} = $d->{$categories}; + print STDERR "TERM NAME - CO IN variable = $d->{$trait_name}\n"; + $variables{$vn}->{$trait_name} = $d->{$trait_name}; + $variables{$vn}->{$trait_id} = $d->{$trait_id}; +} +print STDERR "VARIABLES: ".Dumper(\%variables); + + +my $root_id = format_ontology_id($opt_n, 0); +my $count = $root_id; +my $acc = sprintf "%07d", $count; # the number after the ontology name and a colon + +print STDERR "Starting at term $ontology_name:$acc ...\n"; + +# write obo header +# +print $F <; + +my $root_acc = $ontology_name.":".$acc; +my $root_name = "ROOT"; + +print $F <{$class_id}, + $traits{$k}->{$class_name}, + $traits{$k}->{$class_name}, + undef, + $root_acc, + $root_name, + )."\n"; + + $trait_classes{$k}->{acc} = $class_id; +# $trait_classes{$k}->{name} = $k; + + $count++; + +} + +foreach my $k (sort keys %traits) { + print $F format_trait( + $ontology_name, + $traits{$k}->{$trait_id}, + $traits{$k}->{$trait_name}, + $traits{$k}->{$trait_definition}, + $traits{$k}->{$trait_synonyms}, + $traits{$k}->{$trait_id}, + $traits{$k}->{$trait_class}, # parent trait + )."\n"; + + $traits{$k}->{name} = $traits{$k}->{$trait_name}; + $traits{$k}->{acc} = $traits{$k}->{$trait_id}; + $count++; +} + + + +foreach my $k (sort keys %variables) { + + my $parent_trait = $variables{$k}->{$trait_name}; + my $parent_trait_id = $traits{$variables{$k}->{'Trait - CO'}}->{acc}; + my $parent_trait_name = $traits{ $variables{$k}->{'Trait -CO'}}->{name}; + + print STDERR "VARIABLE: $k. PARENT TRAIT: $parent_trait\n"; + + print $F format_variable( + $ontology_name, + $variables{$k}->{$variable_id}, + $k, ###$variables{$k}->{'Variable Full Name'}, + join(" - ", $variables{$k}->{$trait_definition}), + $variables{$k}->{$trait_synonyms}, + $variables{$k}->{$trait_id}, + $variables{$k}->{$trait_name}, + + #$traits{$variables{$k}->{'Trait - CO'}}->{acc}, # parent trait id + #$traits{$variables{$k}->{'Trait - CO'}}->{name}, # parent trait + + )."\n"; + + print $G format_props( + $k, # variable name + $ontology_name, + $count, + $variables{$k}->{$scale_class}, + $variables{$k}->{$categories}, + ); + + $count++; + +} + +close($F); +close($G); + +print STDERR "Script completed.\n"; + +sub format_props { + my $trait_name = shift; + my $ontology_name = shift; + my $count = shift; + my $trait_format = shift; + my $categories = shift; + + my $trait_default_value = shift; + my $trait_minimum = shift; + my $trait_maximum = shift; + my $trait_details = shift; + + return join ("\t", $trait_name."|".format_ontology_id($ontology_name, $count), $trait_format, $trait_default_value, $trait_minimum, $trait_maximum, $categories, $trait_details)."\n"; + + +} + + +sub format_ontology_id { + my $ontology_name = shift; + my $acc = shift; + + return $ontology_name.":".sprintf "%07d", $acc; +} + +sub format_trait { + my $ontology_code = shift; + my $id = shift; + my $name = shift; + my $description = shift; + my $synonyms = shift; + my $parent_class_id = shift; + my $parent_trait = shift; + + my %record = ( + "[Term]" => "", + "id:" => $id, + "name:" => $name, + "def:" => "\"$description\" []", + "synonym:" => $synonyms, + "namespace:" => $ontology_code, + "is_a:" => "$parent_class_id ! $parent_trait", + ); + + my $data = ""; + foreach my $k ("[Term]", "id:", "name:", "def:", "synonym:", "namespace:", "is_a:") { + if (defined($record{$k})) { + $data .= "$k $record{$k}\n"; + } + } + + return $data; +} + + +sub format_variable { + my $ontology_name = shift; + my $id = shift; + my $name = shift; + my $description = shift; + my $synonyms = shift; + my $parent_trait_id = shift; + my $parent_trait_name = shift; + + #print STDERR "Parent trait name: $parent_trait_name\n"; + +# my $variable_id = format_ontology_id($ontology_code, $id); +# my $parent_trait_id = format_ontology_id($ontology_code, $parent_trait_id); + my %record = ( + "[Term]" => "", + "id:" => $id, + "name:" => $name, + "def:"=> "\"$description\" []", + "synonym:" => $synonyms, + "namespace:" => $ontology_name, + "relationship:" => "variable_of $parent_trait_id ! $parent_trait_name", + ); + + my $data = ""; + foreach my $k ("[Term]", "id:", "name:", "def:", "synonym:", "namespace:", "relationship:") { + if (defined($record{$k})) { + $data .= "$k $record{$k}\n"; + } + } + + return $data; +} diff --git a/bin/convert_td_to_obo.pl b/bin/convert_td_to_obo.pl new file mode 100644 index 0000000000..450589f18c --- /dev/null +++ b/bin/convert_td_to_obo.pl @@ -0,0 +1,260 @@ + +=head1 NAME + +convert_td_to_obo.pl - script to convert trait dictionary file from CO to obo format. Input TD file needs to be in .xlsx format + +=head1 SYNOPSIS + +perl convert_td_to_obo.pl -f trait dictionary file <.xls format> -o output file name <.obo format> + +=head1 DESCRIPTION + +This script currently prints the following fields: + + [Term] + id: $prefix:$accession + name: $cvterm_name + namespace: $namespace + def: "$def" #if the cvterm 'definition' field is populated + is_obsolete: true #if cvterm is_obsolete field is 'true' + synonym: $synonym_name [] # one row for each synonym + xref: $xref_cvterm [] # one row for each cvterm xref that is not for the cvterm is_for_definition + is_a: $cvterm # one row for each is_a relationship object + relationship: $typedef $cvterm # one row for each relationship type that is not is_a (e.g. variable_of, method_of, scale_of) + + -------------------------------------------- + + Example: + + [Term] + id: CO_334:0000009 + name: initial vigor assessment 1-7 + namespace: cassava_trait + def: "Visual assessment of plant vigor during establishment scored one month after planting. 3 = Not vigorous, 5 = Medium vigor, 7 = highly vigorous." [CO:curators] + synonym: "Can't fall when there is strong wind" EXACT [] + synonym: "IVig_IITAVisScg_1to7" EXACT [] + synonym: "vigor" EXACT [] + xref: TO:0000250 + is_a: CO_334:0001000 ! Variables + is_a: CO_334:0002010 ! Farmer trait + relationship: variable_of CO_334:0000386 ! Initial vigor + relationship: variable_of CO_334:0010228 ! Visual Rating: Initial vigor_method + relationship: variable_of CO_334:0100434 ! 7pt scale + + ------------------------------------------ + + +=head1 AUTHOR + +Naama Menda + +=cut + +#!/usr/bin/perl + +use strict; +use warnings; +use Spreadsheet::ParseXLSX; +use Getopt::Std; +use DateTime; +use File::Slurp; + +my %opts; +getopts('f:o:n:m:s:c:p:', \%opts); + +die "Usage: $0 -f -o -p -n -c [crop name] -m [method namespace] -s [scale namespace] \n" + unless $opts{f} && $opts{n} && $opts{p}; + +my $file = $opts{f}; +my $output_file = $opts{o} || "outfile.obo"; +my $trait_namespace = $opts{n} || $opts{c} . "_trait"; +my $method_namespace = $opts{m} || $opts{c} . "_method"; +my $scale_namespace = $opts{s} || $opts{c} . "_scale"; +my $prefix = $opts{p}; + +my $parser = Spreadsheet::ParseXLSX->new(); +my $workbook = $parser->parse($file); + +die "Failed to parse Excel file: $file\n" unless defined $workbook; + +my $worksheet = $workbook->worksheet(0); +die "No worksheet found in the Excel file\n" unless defined $worksheet; + +my ($row_min, $row_max) = $worksheet->row_range(); +my ($col_min, $col_max) = $worksheet->col_range(); + +my ( + $variable_id_col, $variable_name_col, $variable_synonyms_col, $variable_status_col, + $variable_label_col, $variable_def_col, $variable_xref_col, + $trait_id_col, $trait_name_col, $trait_class_col, $trait_def_col, $trait_synonyms_col, $trait_status_col, $trait_xref_col, + $method_id_col, $method_name_col, $method_class_col, $method_def_col, $formula_col, + $scale_id_col, $scale_name_col, $scale_class_col, $lower_limit_col, $upper_limit_col, + $category_1_col, $category_2_col, $category_3_col, $category_4_col, $category_5_col, + $category_6_col, $category_7_col, $category_8_col, $category_9_col, $category_10_col +); + +my %column_map = ( + 'Variable ID' => \$variable_id_col, + 'Variable name' => \$variable_name_col, + 'Variable synonyms' => \$variable_synonyms_col, + 'Variable status' => \$variable_status_col, + 'Variable label' => \$variable_label_col, + 'Variable description' => \$variable_def_col, + 'Variable Xref' => \$variable_xref_col, + 'Trait ID' => \$trait_id_col, + 'Trait name' => \$trait_name_col, + 'Trait class' => \$trait_class_col, + 'Trait description' => \$trait_def_col, + 'Trait synonyms' => \$trait_synonyms_col, + 'Trait status' => \$trait_status_col, + 'Trait Xref' => \$trait_xref_col, + 'Method ID' => \$method_id_col, + 'Method name' => \$method_name_col, + 'Method class' => \$method_class_col, + 'Method description' => \$method_def_col, + 'Formula' => \$formula_col, + 'Scale ID' => \$scale_id_col, + 'Scale name' => \$scale_name_col, + 'Scale class' => \$scale_class_col, + 'Lower limit' => \$lower_limit_col, + 'Upper limit' => \$upper_limit_col, + 'Category 1' => \$category_1_col, + 'Category 2' => \$category_2_col, + 'Category 3' => \$category_3_col, + 'Category 4' => \$category_4_col, + 'Category 5' => \$category_5_col, + 'Category 6' => \$category_6_col, + 'Category 7' => \$category_7_col, + 'Category 8' => \$category_8_col, + 'Category 9' => \$category_9_col, + 'Category 10' => \$category_10_col, +); + +for my $col ($col_min .. $col_max) { + my $cell = $worksheet->get_cell($row_min, $col); + next unless $cell; + my $header = $cell->value(); + $header =~ s/^\s+|\s+\$//g; + if (exists $column_map{$header}) { + ${ $column_map{$header} } = $col; + } +} + +die "Error: 'Variable ID' column not found.\n" unless defined $variable_id_col; +die "Error: 'Variable name' column not found.\n" unless defined $variable_name_col; + +my $trait_class_id = "$prefix:Trait"; +my $method_class_id = "$prefix:Method"; +my $scale_class_id = "$prefix:Scale"; + +my @output; + +ROW_LOOP: +for my $row ($row_min + 1 .. $row_max) { + my $has_data = 0; + for my $col ($col_min .. $col_max) { + my $cell = $worksheet->get_cell($row, $col); + if ($cell && defined $cell->value() && $cell->value() =~ /\S/) { + $has_data = 1; + last; + } + } + last ROW_LOOP unless $has_data; + + + my %cell_values; + for my $header (keys %column_map) { + my $col = ${ $column_map{$header} }; + next unless defined $col; + my $cell = $worksheet->{Cells}[$row][$col]; + $cell_values{$header} = $cell ? $cell->{Val} : undef; + } + + my $variable_id = $cell_values{'Variable ID'}; + #next unless defined $variable_id; + + my $variable_name = $cell_values{'Variable name'}; + my $variable_def = $cell_values{'Variable description'}; + + my $var_synonyms_list = ''; + if (my $variable_synonyms = $cell_values{'Variable synonyms'}) { + my @syns = split /\s*,\s*/, $variable_synonyms; + $var_synonyms_list .= "synonym: \"$_\" EXACT []\n" for @syns; + } + + my $trait_id = $cell_values{'Trait ID'}; + my $trait_name = $cell_values{'Trait name'}; + my $trait_def = $cell_values{'Trait description'}; + + my $trait_synonyms_list = ''; + if (my $trait_synonyms = $cell_values{'Trait synonyms'}) { + my @syns = split /\s*,\s*/, $trait_synonyms; + $trait_synonyms_list .= "synonym: \"$_\" EXACT []\n" for @syns; + } + + my $categories = join(", ", grep { defined } map { $cell_values{"Category $_"} } 1 .. 10); + $categories = "categories: $categories" if $categories; + + my $scale_def = join(", ", grep { defined && $_ ne '' } ( + $cell_values{'Lower limit'} ? "lower_limit: $cell_values{'Lower limit'}" : undef, + $cell_values{'Upper limit'} ? "upper_limit: $cell_values{'Upper limit'}" : undef, + $categories + )); + + # Variable + push @output, "[Term]\n"; + push @output, "id: $variable_id\n"; + push @output, "name: $variable_name\n"; + push @output, "namespace: $trait_namespace\n"; + push @output, "def: \"$variable_def\"\n" if $variable_def; + push @output, $var_synonyms_list if $var_synonyms_list; + push @output, "relationship: variable_of $trait_id ! $trait_name \n"; + push @output, "relationship: variable_of $cell_values{'Method ID'} ! $cell_values{'Method name'} \n" if $cell_values{'Method ID'}; + push @output, "relationship: variable_of $cell_values{'Scale ID'} ! $cell_values{'Scale name'} \n" if $cell_values{'Scale ID'}; + push @output, "xref: $cell_values{'Variable Xref'}\n" if $cell_values{'Variable Xref'}; + push @output, "is_obsolete: true\n\n" if $cell_values{'Variable status'} =~ /^\s*obsolete\s*\$/i; + push @output, "\n"; + + # Trait + push @output, "[Term]\n"; + push @output, "id: $trait_id\n"; + push @output, "name: $trait_name\n"; + push @output, "namespace: $trait_namespace\n"; + push @output, "def: \"$trait_def\"\n" if $trait_def; + push @output, $trait_synonyms_list if $trait_synonyms_list; + push @output, "is_a: $trait_class_id ! $cell_values{'Trait class'} \n"; + push @output, "xref: $cell_values{'Trait Xref'}\n" if $cell_values{'Trait Xref'}; + push @output, "is_obsolete: true\n\n" if $cell_values{'Trait status'} =~ /^\s*obsolete\s*\$/i; + push @output, "\n"; + + # Method + push @output, "[Term]\n"; + push @output, "id: $cell_values{'Method ID'}\n"; + push @output, "name: $cell_values{'Method name'}\n"; + push @output, "namespace: $method_namespace\n"; + push @output, "def: \"$cell_values{'Formula'}\"\n" if $cell_values{'Formula'}; + push @output, "is_a: $method_class_id ! $cell_values{'Method class'} \n\n" if $cell_values{'Method class'}; + + # Scale + push @output, "[Term]\n"; + push @output, "id: $cell_values{'Scale ID'}\n"; + push @output, "name: $cell_values{'Scale name'}\n"; + push @output, "namespace: $scale_namespace\n"; + push @output, "def: \"$scale_def\"\n" if $scale_def; + push @output, "is_a: $scale_class_id ! $cell_values{'Scale class'} \n\n" if $cell_values{'Scale class'}; + +} +my $now = DateTime->now; +my $date_mdy = join(':', $now->month, $now->day, $now->year); +my $time_hm = join(':', $now->hour, sprintf("%02d", $now->minute)); + +my $obo_header = "format-version: 1.2\ndate: $date_mdy $time_hm\ndefault-namespace: $trait_namespace\nontology: $trait_namespace\n\n"; + +my $trait_root_term = "[Term]\nid: $trait_class_id\nname: $trait_namespace ontology\nnamespace: $trait_namespace\n\n"; +my $method_root_term = "[Term]\nid: $method_class_id\nname: methods\nnamespace: $method_namespace\n\n"; +my $scale_root_term = "[Term]\nid: $scale_class_id\nname: scales\nnamespace: $scale_namespace\n\n"; + +write_file($output_file, $obo_header . $trait_root_term . $method_root_term . $scale_root_term); +append_file($output_file, @output); + +print "Conversion to obo complete. Output saved to '$output_file'.\n"; diff --git a/bin/delete_nd_experiment_entries.pl b/bin/delete_nd_experiment_entries.pl index 5a322cde7d..a04869bdfe 100644 --- a/bin/delete_nd_experiment_entries.pl +++ b/bin/delete_nd_experiment_entries.pl @@ -43,11 +43,13 @@ =head1 AUTHOR close($fh); eval { + if ( scalar(@nd_experiment_ids) > 0 ) { my $nd_experiment_ids_string = join ",", @nd_experiment_ids; my $q = "DELETE FROM nd_experiment WHERE nd_experiment_id IN ($nd_experiment_ids_string)"; my $h = $dbh->prepare($q); $h->execute(); - print STDERR "DELETED ".scalar(@nd_experiment_ids)." Nd Experiment Entries\n"; + } + print STDERR "DELETED ".scalar(@nd_experiment_ids)." Nd Experiment Entries\n"; }; if ($@) { diff --git a/bin/delete_stocks.pl b/bin/delete_stocks.pl index b15cbbd8f4..8a5b2f5133 100644 --- a/bin/delete_stocks.pl +++ b/bin/delete_stocks.pl @@ -7,15 +7,22 @@ =head1 DESCRIPTION perl delete_stocks.pl -H [host] -D [dbname] [-s accession ] -t (for testing) file -where the file contains a list of uniquenames specifying the stocks to be deleted, one per line. +where the file is a text file containing a list of uniquenames specifying the stocks to be deleted, one per line. + +The script will check now if the stock has any associated experiments and not delete such accessions. + +The parameter -s specifies the type of stock (accession, tissue_sample, or plant) to be deleted. -The parameter -s specifies the type of stock (accession, plot, family_name, etc) to be deleted. The default is "accession". +Note that the script cannot delete plots, subplots, crosses or families. This is because these data types are managed on a trial or cross level and should not be individually modified. + If the -t flag is provided, the changes will be rolled back in the database. Note that it may be possible that some stocks have additional connections, such as images, that this script does not delete yet, and so won't be able to delete those stocks. +The file is a text file containing one accession name per line. There is no header line. + =head1 AUTHOR Lukas Mueller @@ -27,18 +34,30 @@ =head1 AUTHOR use DBI; use Bio::Chado::Schema; use CXGN::Phenome::Schema; +use CXGN::Stock; +use CXGN::Stock::Plot; -our ($opt_H, $opt_D, $opt_t, $opt_s); +our ($opt_H, $opt_D, $opt_t, $opt_s, $opt_p); -getopts('H:D:s:t'); +getopts('H:D:s:tp:'); my $stock_type = $opt_s || "accession"; +if ($stock_type eq 'plot' || $stock_type eq 'subplot' || $stock_type eq 'cross' || $stock_type eq 'family' ) { + print STDERR "This script cannot delete plots or subplots. They have to be managed through the trial or cross interface.\n"; + exit(); +} + my $file = shift; -print "Password for $opt_H / $opt_D: \n"; -my $pw = <>; -chomp($pw); +my $pw; + +if (! $opt_p) { + print "Password for $opt_H / $opt_D: \n"; + $pw = <>; + chomp($pw); +} +else { $pw = $opt_p; } print STDERR "Connecting to database...\n"; my $dsn = 'dbi:Pg:database='.$opt_D.";host=".$opt_H.";port=5432"; @@ -62,63 +81,47 @@ =head1 AUTHOR while (<$F>) { chomp; - my $stock = $_; - $stock =~ s/\r//g; - if (!$stock) { + my $stock_name = $_; + $stock_name =~ s/\r//g; + if (!$stock_name) { next(); } $stock_count++; - print STDERR "Processing $stock\n"; + print STDERR "Processing $stock_name\n"; - my $stock_row = $bcs_schema->resultset("Stock::Stock")->find( { uniquename => $stock, type_id => $stock_type_cvterm_id }); + my $stock_row = $bcs_schema->resultset("Stock::Stock")->find( { uniquename => $stock_name, type_id => $stock_type_cvterm_id }); if (!$stock_row) { - print STDERR "Could not find stock $stock of type $stock_type. Skipping...\n"; + print STDERR "Could not find stock $stock_name of type $stock_type. Skipping...\n"; $missing_stocks++; next; } + + my $stock_id = $stock_row->stock_id(); + + my $stock = CXGN::Stock->new( { schema => $bcs_schema, stock_id => $stock_id }); - my $owner_rs = $phenome_schema->resultset("StockOwner")->search( { stock_id => $stock_row->stock_id() }); - if ($owner_rs->count() > 1) { - print STDERR "Weird. $stock has more than one owner.\n"; - } - - my $subject_relationship_rs = $bcs_schema->resultset("Stock::StockRelationship")->search( { object_id => $stock_row->stock_id() }); + # check if stock has associated trials, refuse to delete if yes + # + my @trials = $stock->get_trials(); - while (my $r = $subject_relationship_rs->next()) { - print STDERR "Found object relationship with stock ".$r->subject_id()." of type ".$r->type_id()."\n"; - } - - my $object_relationship_rs = $bcs_schema->resultset("Stock::StockRelationship")->search( { subject_id => $stock_row->stock_id() }); - while (my $r = $object_relationship_rs->next()) { - print STDERR "Found subject relationship with stock ".$r->object_id()." of type ".$r->type_id()."\n"; - } + if (@trials > 0) { + print STDERR "Stock $stock_name cannot be deleted because it is associated with trials ".join(", ", map { $_->[1] } @trials).". Skipping...\n"; - while (my $owner_row = $owner_rs->next()) { - - if (! $opt_t) { - eval { - print STDERR "Removing stockowner (".$owner_row->stock_id().")...\n"; - $owner_row->delete(); - }; - if ($@) { - print STDERR "Could not delete owner of stock $stock because of: $@\stock"; - } - } - - $stock_owner_count++; + next(); } if (! $opt_t) { eval { - $stock_row->delete(); + $stock->hard_delete(); }; if ($@) { print STDERR "Could not delete entry for stock $stock because of: $@\n"; } - else { + else { + print STDERR "Successfully deleted stock $stock_name\n"; $deleted_stock_count++; } } diff --git a/bin/delete_trials.pl b/bin/delete_trials.pl index 0771a686c1..3d2105655c 100644 --- a/bin/delete_trials.pl +++ b/bin/delete_trials.pl @@ -5,9 +5,50 @@ =head1 NAME =head1 DESCRIPTION -perl delete_trials.pl -i trial_id -H host -D dbname -U dbuser -P dbpass -b basepath -r temp_file_nd_experiment_id +perl delete_trials.pl -i trial_ids [ -t trial_names ] [ -F file_with_trial_names ] [ -f file_with_trial_ids ] -H host -D dbname -U dbuser -P dbpass -b basepath -r temp_file_nd_experiment_id [ -n ] + +Options: + +=over 5 + +=item -H + +hostname for database + +=item -D + +database name + +=item -i + +comma separated list of trial ids + +=item -t + +comma separated list of trial names + +=item -b + +basebath is the install path of the software, most commonly /home/production/cxgn/sgn which is the default. + +=item -r + +Specifies the temp file used to track nd_experiment_ids to delete. Defaults to /tmp/temp_nd_experiment_id_[date_and_time]. + +=item -F + +a file with trial names, one per line + +=item -f + +file with trial ids, one per line + +=item -n + +non-interactive mode. Will not prompt for confirmation of each trial to delete + +=back -Deletes trials that whose ids are provided as a comma separated list for the -i parameter. First, it deletes metadata, then trial layouts, then phenotypes, and finally the trial entry in the project table. All deletes are hard deletes. There is no way of bringing the trial back, except from a backup. So be careful! =head1 AUTHOR @@ -19,23 +60,30 @@ =head1 AUTHOR use strict; use Getopt::Std; +use DateTime; use Bio::Chado::Schema; use CXGN::Metadata::Schema; use CXGN::Phenome::Schema; use CXGN::DB::InsertDBH; use CXGN::Trial; -our ($opt_H, $opt_D, $opt_U, $opt_P, $opt_b, $opt_i, $opt_n, $opt_t, $opt_r); +our ($opt_H, $opt_D, $opt_U, $opt_P, $opt_b, $opt_i, $opt_t, $opt_n, $opt_r, $opt_F, $opt_f); -getopts('H:D:U:P:b:i:t:r:n'); +getopts('H:D:U:P:b:i:t:r:nf:F:'); +my $dt = DateTime->now(); +my $date_string = $dt->ymd()."T".$dt->hms(); my $dbhost = $opt_H; my $dbname = $opt_D; my $dbuser = $opt_U; my $dbpass = $opt_P; my $trial_ids = $opt_i; my $trial_names = $opt_t; +my $trial_names_file = $opt_F; +my $trial_ids_file = $opt_f; my $non_interactive = $opt_n; +my $basepath = $opt_b || '/home/production/cxgn/sgn'; +my $tempfile = $opt_r || "/tmp/temp_nd_experiment_id_$date_string"; my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost, dbname=>$dbname, @@ -52,43 +100,71 @@ =head1 AUTHOR my @trial_ids = split ",", $trial_ids; my @trial_names = split ",", $trial_names; +if ($trial_names_file) { + open(my $F, "<", $trial_names_file) || die "Can't open the file $trial_names_file"; + while (<$F>) { + chomp; + push @trial_names, $_; + } + close($F); +} + +if ($trial_ids_file) { + open(my $F, "<", $trial_ids_file) || die "Can't open the file $trial_ids_file"; + while(<$F>) { + chomp; + push @trial_ids, $_; + } + close($F); +} + foreach my $name (@trial_names) { my $trial = $schema->resultset("Project::Project")->find( { name => $name }); if (!$trial) { print STDERR "Trial $name not found. Skipping...\n"; next; } push @trial_ids, $trial->project_id(); } -foreach my $trial_id (@trial_ids) { - print STDERR "Retrieving trial information for trial $trial_id...\n"; - - my $t = CXGN::Trial->new({ - bcs_schema => $schema, - metadata_schema => $metadata_schema, - phenome_schema => $phenome_schema, - trial_id => $trial_id - }); +eval { + $dbh->do("set search_path to public,sgn,phenome,sgn_people,metadata"); + my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() }); - my $answer = ""; - if (!$non_interactive) { - print $t->get_name().", ".$t->get_description().". Delete? "; - $answer = <>; - } - if ($non_interactive || $answer =~ m/^y/i) { - eval { - delete_trial($metadata_schema, $phenome_schema, $t); - }; - if ($@) { - print STDERR "An error occurred trying to delete trial ".$t->get_name()." ($@)\n"; - $dbh->rollback(); + my $metadata_schema = CXGN::Metadata::Schema->connect( sub { $dbh->get_actual_dbh() }); + my $phenome_schema = CXGN::Phenome::Schema->connect( sub { $dbh->get_actual_dbh() }); + + foreach my $trial_id (@trial_ids) { + print STDERR "Retrieving trial information for trial $trial_id...\n"; + + my $t = CXGN::Trial->new({ + bcs_schema => $schema, + metadata_schema => $metadata_schema, + phenome_schema => $phenome_schema, + trial_id => $trial_id + }); + + my $answer = ""; + if (!$non_interactive) { + print $t->get_name().", ".$t->get_description().". Delete? "; + $answer = <>; } - else { - $dbh->commit(); - print STDERR "Trial ".$t->get_name()." successfully deleted\n"; + if ($non_interactive || $answer =~ m/^y/i) { + + delete_trial($metadata_schema, $phenome_schema, $t); + } - } +}; +if ($@) { + print STDERR "ERROR: $@\n"; + $dbh->rollback(); } +else { + $dbh->commit(); + print STDERR "Trials successfully deleted\n"; +} + + + $dbh->disconnect(); print STDERR "Done with everything (though nd_experiment entry deletion may still be occuring asynchronously).\n"; @@ -101,7 +177,7 @@ sub delete_trial { print STDERR "Deleting trial ".$t->get_name()."\n"; print STDERR "Delete metadata...\n"; $t->delete_metadata(); - print STDERR "Deleting phenotypes...\n"; + print STDERR "Deleting phenotypes... (using $dbhost, $dbuser, $dbname)\n"; $t->delete_phenotype_data($opt_b, $dbhost, $dbname, $dbuser, $dbpass, $opt_r); print STDERR "Deleting layout...\n"; $t->delete_field_layout(); diff --git a/bin/download_obo.pl b/bin/download_obo.pl new file mode 100644 index 0000000000..2ceae88b0c --- /dev/null +++ b/bin/download_obo.pl @@ -0,0 +1,185 @@ + +=head1 NAME + +download_obo.pl - script to download obo file of an ontology + +=head1 SYNOPSIS + +perl download_obo.pl -i prefix -H host -D dbname + +=head1 DESCRIPTION + +Download an obo file from your database. The file will contain all terms listed under the provided ontology prefix (db.name). +The obo filename is $prefix_breedbase.obo. +After downloading the file it is recommended to test it in the Protege program to make sure it is a valid obo, +and loading back into the database with gmod_load_cvterms.pl + + +This script currently prints the following fields: + + [Term] + id: $prefix:$accession + name: $cvterm_name + namespace: $namespace + def: "$def" #if the cvterm 'definition' field is populated + is_obsolete: true #if cvterm is_obsolete field is 'true' + synonym: $synonym_name [] # one row for each synonym + xref: $xref_cvterm [] # one row for each cvterm xref that is not for the cvterm is_for_definition + is_a: $cvterm # one row for each is_a relationship object + relationship: $typedef $cvterm # one row for each relationship type that is not is_a (e.g. variable_of, method_of, scale_of) + + -------------------------------------------- + + Example: + + [Term] + id: CO_334:0000009 + name: initial vigor assessment 1-7 + namespace: cassava_trait + def: "Visual assessment of plant vigor during establishment scored one month after planting. 3 = Not vigorous, 5 = Medium vigor, 7 = highly vigorous." [CO:curators] + synonym: "Can't fall when there is strong wind" EXACT [] + synonym: "IVig_IITAVisScg_1to7" EXACT [] + synonym: "vigor" EXACT [] + xref: TO:0000250 + is_a: CO_334:0001000 ! Variables + is_a: CO_334:0002010 ! Farmer trait + relationship: variable_of CO_334:0000386 ! Initial vigor + relationship: variable_of CO_334:0010228 ! Visual Rating: Initial vigor_method + relationship: variable_of CO_334:0100434 ! 7pt scale + + ------------------------------------------ + +If there are any cvterm.is_relationshiptype for this ontology they will be printed as + [Typedef] + id: $cvterm_name + name: $cvterm_name + + +=head1 COMMAND-LINE OPTIONS + + -H host name + -D database name + -i prefix for the ontology (e.g. CO_334) + -t Test run. Rolls back at the end. + +=head1 AUTHOR + +Naama Menda + +=cut + +use strict; + +use Getopt::Std; +use Data::Dumper; +use Try::Tiny; +use DateTime; + +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; +use File::Slurp; + +our ($opt_H, $opt_D, $opt_U, $opt_P, $opt_i); + +getopts('H:D:U:P:i:'); + +my $dbhost = $opt_H; +my $dbname = $opt_D; +my $dbuser = $opt_U; +my $dbpass = $opt_P; +my $prefix = $opt_i; + +my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 0, + RaiseError => 1} + } ); + +print STDERR "Connecting to database...\n"; +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); + +my $obo_file = $prefix . ".breedbase.obo"; + +#resultset of all cvterms +my $cvterm_rs = $schema->resultset("Cv::Cvterm")->search( + { 'db.name' => $prefix }, + { join => [ 'cv', { dbxref => 'db' } ] , + '+select' => ['cv.name', 'dbxref.accession'], + '+as' => [ 'cv_name', 'dbxref_accession' ] +} ); + +my $date = DateTime->now(); + +my $obo_header = "format-version: 1.2 +ontology: $prefix\n\n"; + +write_file( $obo_file, {append => 0 }, $obo_header ) ; +my $count=0; +while(my $cvterm = $cvterm_rs->next() ) { + my $accession = $cvterm->dbxref->accession(); + print STDERR "Looking at Accession $accession\n"; + my $cvterm_name = $cvterm->name(); + my $namespace = $cvterm->cv->name(); + my $def = $cvterm->definition(); + #remove quotes from definition sting + $def =~ s/"//g; + my $is_obsolete = $cvterm->is_obsolete(); + my $is_relationshiptype = $cvterm->is_relationshiptype(); + my $is_obsolete = $cvterm->is_obsolete(); + + my $term_details = "\n[Term]\nid: $prefix:$accession\nname: $cvterm_name\nnamespace: $namespace\n"; + $term_details .="def: \"$def\"\n" if $def; + $term_details .="is_obsolete: true\n" if $is_obsolete; + $count++; + if ($is_relationshiptype) { + $term_details = " +[Typedef] +id: $cvterm_name +name: $cvterm_name +" + } + write_file( $obo_file, {append => 1 }, "$term_details" ); + + my $syn_rs = $cvterm->cvtermsynonyms(); + my $xref_rs = $cvterm->cvterm_dbxrefs(); + my $relationships_rs = $cvterm->cvterm_relationship_subjects(); + + while( my $synonym = $syn_rs->next() ) { + my $syn_name = $synonym->synonym(); + print STDERR "synonym = $syn_name\n"; + my $type = $synonym->type; + my $type_name ; + + defined $type ? $type_name = $type->name : "[]"; + + #synonyms need to be quoted. Somtimes they are already quoted if loaded properly from the obo loader + unless ($syn_name =~ /^\"/) { $syn_name = '"' . $syn_name . '"' ; } + #xref list for synonyms + unless ($syn_name =~ /$\]/) { $syn_name .= " []" ; } + write_file( $obo_file, {append => 1 }, "synonym: " . $syn_name . $type_name . "\n" ); + } + while( my $xref = $xref_rs->next() ) { + my $xref_acc = $xref->dbxref->accession(); + my $xref_prefix = $xref->dbxref->db->name(); + print STDERR "xref = $xref_prefix:$xref_acc\n"; + write_file( $obo_file, {append => 1 }, "xref: " . $xref_prefix . ":" . $xref_acc . "\n" ) if ( $xref->is_for_definition == 0 ); + } + + while( my $rel = $relationships_rs->next() ) { + my $object = $rel->object(); + my $type = $rel->type(); + my $object_name = $object->name(); + my $object_acc = $object->dbxref->accession(); + my $object_acc_prefix = $object->dbxref->db->name(); + + my $type_name = $type->name(); + + my $relationship_format = "is_a:"; + if ($type_name ne "is_a") { + $relationship_format = "relationship: $type_name"; + } + print STDERR "$relationship_format $object_acc_prefix:$object_acc\n"; + write_file( $obo_file, {append => 1 }, "$relationship_format $object_acc_prefix:$object_acc " . "! ". $object_name . "\n" ); + } +} +print STDERR "wrote $count terms to file $obo_file\n"; diff --git a/bin/download_spatial_trials.pl b/bin/download_spatial_trials.pl new file mode 100644 index 0000000000..112c775430 --- /dev/null +++ b/bin/download_spatial_trials.pl @@ -0,0 +1,222 @@ + +=head1 NAME + +download_trials.pl - script to download trials + +=head1 DESCRIPTION + +perl download_spatial_trials.pl -H host (breedbase_db) -D dbname (cxgn_cassava) -U dbuser (postgres) -P dbpass (postgres) + +Downloads trials whose ids are provided as a comma separated list for the -i parameter. + +=head1 AUTHOR + +Lukas Mueller + +=cut + +use strict; + +use Getopt::Std; +use Data::Dumper; + +use Bio::Chado::Schema; +use CXGN::Dataset::File; +use CXGN::Metadata::Schema; +use CXGN::Phenome::Schema; +use CXGN::DB::InsertDBH; +use CXGN::Trial; + +our ($opt_H, $opt_D, $opt_U, $opt_P); + +getopts('H:D:U:P'); + +my $dbhost = $opt_H; +my $dbname = $opt_D; +my $dbuser = $opt_U; +my $dbpass = $opt_P; +#my $non_interactive = $opt_n; + +my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 0, + RaiseError => 1} + } + ); + +print STDERR "Connecting to database...\n"; +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); +my $metadata_schema = CXGN::Metadata::Schema->connect( sub { $dbh->get_actual_dbh() }); +my $phenome_schema = CXGN::Phenome::Schema->connect( sub { $dbh->get_actual_dbh() }); +my $people_schema = CXGN::People::Schema->connect( sub { $dbh->get_actual_dbh() } ); + +print STDERR "Retrieving all projects...\n"; +my $trial_rs = $schema->resultset("Project::Project")->search( { } ); # get all trials + +my @trial_ids; + +while (my $row= $trial_rs->next()) { + print STDERR "Retrieving project row with id ".$row->project_id()."\n"; + push @trial_ids, $row->project_id(); +} + + +my @spreadsheet; +my %trial_data; +my %trial_cols; + +foreach my $trial_id (@trial_ids) { + print STDERR "Retrieving trial information for trial $trial_id...\n"; + + my $t = CXGN::Trial->new({ + bcs_schema => $schema, + metadata_schema => $metadata_schema, + phenome_schema => $phenome_schema, + trial_id => $trial_id + }); + + if ($t->isa("CXGN::PhenotypingTrial")) { + print STDERR "We have a field trial!\n"; + } + else { + print STDERR "Trial with id $trial_id is not a field trial. Skipping.\n"; + next(); + } + + if (! $t->has_col_and_row_numbers()) { + print STDERR "Trial with id $trial_id does not have a spatial layout. Skipping.\n"; + next(); + } + + # retrieve associated genotypes using a dataset + # + my $d = CXGN::Dataset::File->new( { people_schema => $people_schema, schema => $schema } ); + + $d->trials( [ $trial_id ]); + + my $genotyping_protocols = $d->retrieve_genotyping_protocols(); + + if (ref($genotyping_protocols) && (@$genotyping_protocols > 0) ) { + + my $accessions = $d->retrieve_accessions(); + + print STDERR "Genotyping protocols: ". Dumper($genotyping_protocols). Dumper($accessions); + } + else { + print STDERR "This trial has no genotyping info associated with it. Skipping!\n"; + next(); + } + + my $location = $t->get_location(); + + my $breeding_programs = $t->get_breeding_programs(); + + my $breeding_program_name = $t->get_breeding_program(); + + my $planting_date = $t->get_planting_date(); + + my $harvest_date = $t->get_harvest_date(); + + my $breeding_program_id; + my $breeding_program_description; + + foreach my $bp (@$breeding_programs) { + if ($bp->[1] eq $breeding_program_name) { + $breeding_program_id = $bp->[0]; + $breeding_program_description = $bp->[2]; + } + } + + my $trial_name = $t->get_name(); + + my $traits = $t->get_traits_assayed(); + + my $year = $t->get_year(); + + my $trial_id = $t->get_trial_id(); + + my $design_type = $t->get_design_type(); + + my $plot_width = $t->get_plot_width(); + + my $plot_length = $t->get_plot_length(); + + print STDERR "Traits assayed = ".Dumper($traits); + + my @trait_names = map { $_->[1] } @$traits; + my @trait_ids = map { $_->[0] } @$traits; + + print STDERR "trait_ids = ". Dumper(\@trait_ids); + + my $data = $t->get_stock_phenotypes_for_traits(\@trait_ids, 'all', ['plot_of','plant_of'], 'accession', 'subject'); + + print STDERR Dumper($data); + + + $trial_data{$trial_id} = $data; + $trial_cols{$trial_id} = [ $year, $breeding_program_id, $breeding_program_name, $breeding_program_description, $trial_id, $trial_name, $design_type, $plot_width, $plot_length, '', '', '', $planting_date, $harvest_date, $location->[0], $location->[1] ]; + + +} + +my @trial_header = qw | studyYear programDbId breeding_programName programDescription studyDbId studyName studyDesign plotWidth plotLength fieldSize fieldTrialIsPlannedToBeGenotyped fieldTrialIsPlannedToCross plantingDate harvestDate, locationDbId, locationName |; + +# first organize traits in hash structure +my %obs; +my %traits; +my %plots; +my %plot_ids; +foreach my $trial_id (keys %trial_data) { + foreach my $line (@{$trial_data{$trial_id}}) { + # keys: {trial_id} -> {accession}-> {plot} -> {trait} = value + $obs{$trial_id}->{$line->[9]}->{$line->[1]}->{$line->[3]} = $line->[7]; + $traits{$line->[3]}++; + $plots{$line->[1]} = $line->[0]; + } + +} + + +# get plot metadata + +my %plot_data; +foreach my $p (keys %plots) { + my $rs= $schema->resultset("Stock::Stockprop")->search( { stock_id => $plots{$p} }, { join => 'type', '+select' => 'type.name', '+as'=> 'cvterm_name' }); + + while (my $row = $rs->next()) { + print STDERR "stockprop: ".$row->get_column("cvterm_name"). " ".$row->value()."\n"; + $plot_data{$p}->{$row->get_column("cvterm_name")} = $row->value(); + } +} + + +print STDERR "observations: ".Dumper(\%obs); + +print STDERR "Traits: ".Dumper(\%traits); + +print join("\t", (@trial_header, 'accession', 'plot', 'replicate', 'blockNumber', 'plotNumber', 'rowNumber', 'colNumber', 'entryType', sort(keys(%traits))))."\n"; + +foreach my $trial_id (keys(%obs)) { + + foreach my $accession (keys %{$obs{$trial_id}}) { + + foreach my $plot (keys %{$obs{$trial_id}->{$accession}}) { + + + my @out = ( @{$trial_cols{$trial_id}}, $accession, $plot ); + + foreach my $prop (qw| replicate block_number plot_number row_number col_number entry_type |) { + push @out, $plot_data{$plot}->{$prop}; + } + + + foreach my $trait (sort(keys %traits)) { + push @out, $obs{$trial_id}->{$accession}->{$plot}->{$trait}; + } + print join("\t", @out)."\n"; + } + } +} + + +#print STDERR "spreadsheet : ". Dumper(\@spreadsheet); diff --git a/bin/drop_test_databases.pl b/bin/drop_test_databases.pl index 5ff376db82..7383374430 100644 --- a/bin/drop_test_databases.pl +++ b/bin/drop_test_databases.pl @@ -3,6 +3,10 @@ =head1 NAME drop_test_databases.pl - a script to remove test databases from the postgres instance +=head1 SYNOPSYS + +perl drop_test_databases.pl -h dbhost + =head1 DESCRIPTION When using t/test_fixture.pl with the --nocleanup option, a lot of test databases (named test_db_* ) can accumulate in the postgres instance. diff --git a/bin/extract_pheno_zeros.pl b/bin/extract_pheno_zeros.pl new file mode 100644 index 0000000000..a9840116ae --- /dev/null +++ b/bin/extract_pheno_zeros.pl @@ -0,0 +1,120 @@ +#!/usr/bin/perl + +=head1 NAME + +extract_pheno_zeros.pl - find all 0 phenotype values in archived uploads and generate a CSV file to re-upload + +=head1 DESCRIPTION + +extract_pheno_zeros.pl -H [database host] -D [database name] -U [database uesr] -P [database pass] -s [start date YYYY-MM-DD] -o [output csv file] + +Options: + + -H the database host + -D the database name + -U username + -P password + -d start date YYYY-MM-DD (default = 2024-06-11) + -o output .csv file + +=head1 AUTHOR + +David Waring + +=cut + +use strict; +use warnings; +use DBI; +use Try::Tiny; +use Getopt::Long; +use Data::Dumper; +use CXGN::File::Parse; + +my ( $dbhost, $dbname, $username, $password, $date, $output ); +GetOptions( + 'H=s' => \$dbhost, + 'D=s' => \$dbname, + 'U=s' => \$username, + 'P=s' => \$password, + 's=s' => \$date, + 'o=s' => \$output, +); + +if ( !$dbhost || !$dbname || !$username || !$password ) { + print STDERR "ERROR: Missing either -H dbhost -D dbname -U username or -P password\n"; + exit 1; +} +if ( !$output ) { + print STDERR "ERROR: Missing output .xls file\n"; + exit 1; +} +if ( !$date || $date eq '' ) { + $date = "2024-06-11"; +} + +print STDERR "Connecting to database...\n"; +my $dsn = 'dbi:Pg:database='.$dbname.";host=".$dbhost.";port=5432"; +my $dbh = DBI->connect($dsn, $username, $password, { RaiseError => 1, AutoCommit=>0 }); + +# Get all phenotype files from the metadata since the start date +my $q = "SELECT dirname || '/' || basename +FROM metadata.md_files +LEFT JOIN metadata.md_metadata ON (md_files.metadata_id = md_metadata.metadata_id) +WHERE md_files.filetype = 'spreadsheet phenotype file' AND md_metadata.create_date > ? +ORDER BY create_date ASC;"; +my $h = $dbh->prepare($q); +$h->execute($date); +$dbh->commit(); + +my %data; +my %traits; + +# Check each file for 0s +while ( my ($file) = $h->fetchrow_array() ) { + print STDERR "==> Checking File: $file\n"; + my $parser = CXGN::File::Parse->new( + file => $file, + required_columns => [ 'observationunit_name' ], + column_aliases => { + 'observationunit_name' => [ 'plot_name', 'subplot_name', 'plant_name', 'observationUnitName', 'plotName', 'subplotName', 'plantName' ] + } + ); + my $parsed = $parser->parse(); + my $parsed_data = $parsed->{data}; + my $trait_columns = $parsed->{optional_columns}; + + foreach my $row (@$parsed_data) { + my $ou = $row->{'observationunit_name'}; + foreach my $trait (@$trait_columns) { + my $value = $row->{$trait}; + if ( defined($value) && $value eq '0' ) { + print STDERR "$ou | $trait = 0\n"; + $traits{$trait} = 1; + $data{$ou}{$trait} = '0'; + } + } + } +} + +# Generate output CSV data +my @output; +push @output, join(',', 'observationunit_name', keys %traits); +my @ous = sort keys %data; +foreach my $ou (@ous) { + my @line; + push @line, $ou; + foreach my $trait (keys %traits) { + my $value = $data{$ou}{$trait}; + push @line, defined($value) ? $value : ''; + } + push @output, join(',', @line); +} + + +# Write CSV to file +open my $fh, '>', $output or die "Cannot open output file: $!"; +foreach (@output) { + print $fh "$_\n"; +} +close $fh; diff --git a/bin/filter_imputed_genotype.pl b/bin/filter_imputed_genotype.pl new file mode 100644 index 0000000000..99a75cb189 --- /dev/null +++ b/bin/filter_imputed_genotype.pl @@ -0,0 +1,119 @@ +#!/usr/bin/perl + + +=head1 +filter_imputed_genotype.pl - filters SNPs based on DR2 and MAF cutoffs from genotype data file. + +=head1 SYNOPSIS +perl filter_imputed_genotype.pl -i unfiltered.vcf -o filtered.vcf -d 0.75 -f 0.005 -h 1e-20 + +=head1 COMMAND-LINE OPTIONS +-i vcf file with imputed (using beagle 5.0 or later) genotype data. Required. +-o vcf file to write filtered output to. Optional. By default it writes to a file ending in _filtered.vcf +-d DR2 (dosage R-squared) threshold. SNPs with greater than the threshold will be kept. Defaults to 0.75. Optional. +-f MAF (minor allele frequency) threshold. SNPs with greater than the threshold will be kept. Defaults to 0.005. Optional. +-h HWE (Hardy Weinberg Equilibrium) threshold. NOT used for now. + +=head1 DESCRITPION +Filters SNPs based on DR2 and MAF cutoffs from genotype data. +Requires that the file is a vcf file and in the INFO column there are DR2 and AF. +It generates two files, one with SNPs that have DR2 and MAF above the cutoffs (*_filtered.vcf) and another file (*_removed.vcf) +with the ones below the thresholds of either DR2 or MAF. + +=head1 AUTHOR +Isaak Y Tecle + +=cut + +use strict; + +use Getopt::Std; +use File::Slurp qw /read_file write_file/; + + +our($opt_i, $opt_o,$opt_d, $opt_f, $opt_h); + +getopts('i:d:f:h:o:'); + +if (!$opt_i || -s $opt_i < 1 ) { +die "Either the imputed genotype file is not provided or is empty."; +} + +my $maf_cutoff = $opt_f ? $opt_f : 0.005; +my $dr2_cutoff = $opt_d ? $opt_d : 0.75; + +my $file_prefix = $opt_i =~ s/\.vcf//r; + +if (!$opt_o) { + $opt_o = $file_prefix . "_filtered.vcf"; +} + +my $kept_snps_file = $opt_o; +my $removed_snps_file = $file_prefix . "_removed.vcf"; + +open(my $I, "<", $opt_i) || die "Can't open imputed vcf_file: $opt_i\n"; + +my $info_idx = 7; +my $format_idx = 8; + +my $removed_snp_cnt = 0; +my $kept_snp_cnt = 0; +my $total_snp_cnt = 0; +my $removed_snps; +my $kept_snps; + +while (<$I>) { + if ($_ =~ /#/) { + $removed_snps .= $_; + $kept_snps .= $_; + } else { + $total_snp_cnt++; + my @cols = split(/\t/, $_); + my $info_col = $cols[$info_idx]; + # my $format_col = $cols[$format_idx]; + + my @info = split(/;/, $info_col); + my ($dr2) = grep(/DR2=\d+\.\d+/, @info); + my ($af) = grep(/AF=\d+\.\d+/, @info); + print STDERR "\nDR2: $dr2 -- AF: $af\n"; + + if (!$dr2 && !$af) { + die "\nThe INFO column has no DR2 and AF values. Stopping filtering attempt.\n"; + } + + if (!$dr2) { + warn "\nThe INFO has no DR2 values. DR2 filter will not be applied\n"; + } + + if (!$af) { + warn "\nThe INFO has no AF values. MAF filter will not be applied\n"; + } + + + $dr2 =~ s/DR2=//; + $af =~ s/AF=//; + + my $maf = $af < 0.5 ? $af : 1 - $af; + + if ($dr2 < $dr2_cutoff || $maf < $maf_cutoff) { + print STDERR "\nREMOVING SNP $cols[1] --info: $info_col --DR2: $dr2 --AF: $af --MAF: $maf "; + $removed_snp_cnt++; + $removed_snps .= $_; + } else { + print STDERR "\nKEEPING SNP $cols[1] --info: $info_col --DR2: $dr2 --AF: $af --MAF: $maf"; + $kept_snp_cnt++; + $kept_snps .= $_; + } + + } +} + +print STDERR "\n\nFiltering criteria: Remove SNPs with DR2 < $dr2_cutoff or MAF < $maf_cutoff"; +print STDERR "\n\nRemoved $removed_snp_cnt SNPs out of $total_snp_cnt."; +print STDERR "\nKept $kept_snp_cnt SNPs out of $total_snp_cnt."; + +write_file($kept_snps_file, $kept_snps); +write_file($removed_snps_file, $removed_snps); +print STDERR"\n\nWrote filtered (kept) SNPs to $kept_snps_file"; +print STDERR"\nWrote removed SNPs to $removed_snps_file"; +print STDERR"\nDone!\n\n"; \ No newline at end of file diff --git a/bin/find_accessions_for_seedlots.pl b/bin/find_accessions_for_seedlots.pl new file mode 100644 index 0000000000..4fed4d03e1 --- /dev/null +++ b/bin/find_accessions_for_seedlots.pl @@ -0,0 +1,131 @@ +#!/usr/bin/perl + +=head1 + +find_accessions_for_seedlots.pl - give the accession for each seedlot name + +=head1 SYNOPSIS + + find_accessions_for_seedlots.pl -H [dbhost] -D [dbname] -i [infile] + +=head1 COMMAND-LINE OPTIONS + ARGUMENTS + -H host name (required) e.g. "localhost" + -D database name (required) e.g. "cxgn_cassava" + -i path to infile (required) + +=head1 DESCRIPTION + + + +=head1 AUTHOR + + Lukas Mueller , based on a script by Nick Morales + +=cut + +use strict; + +use Getopt::Std; +use Data::Dumper; +use Carp qw /croak/ ; +use Pod::Usage; +use Spreadsheet::ParseExcel; +use Spreadsheet::ParseXLSX; +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; +use SGN::Model::Cvterm; +use CXGN::Stock; +use CXGN::Stock::Seedlot; + +our ($opt_H, $opt_D, $opt_i); + +getopts('H:D:i:'); + +if (!$opt_H || !$opt_D || !$opt_i) { + pod2usage(-verbose => 2, -message => "Must provide options -H (hostname), -D (database name), -i (input file) \n"); +} + +my $dbhost = $opt_H; +my $dbname = $opt_D; + +# Match a dot, extension .xls / .xlsx +my ($extension) = $opt_i =~ /(\.[^.]+)$/; +my $parser; + +print STDERR "Figuring out file format... ($extension)...\n"; +if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); +} +else { + $parser = Spreadsheet::ParseExcel->new(); +} + +#print STDERR "Parsing file... (please wait...)\n"; + +#my $excel_obj = $parser->parse($opt_i); + +print STDERR "Connecting to database $dbname on $dbhost ...\n"; + +my $dbh = CXGN::DB::InsertDBH->new({ + dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 1, RaiseError => 1} +}); + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); +$dbh->do('SET search_path TO public,sgn'); + + +print STDERR "Parsing file...\n"; + +#my $worksheet = ( $excel_obj->worksheets() )[0]; #support only one worksheet +#my ( $row_min, $row_max ) = $worksheet->row_range(); +#my ( $col_min, $col_max ) = $worksheet->col_range(); + +my $seedlot_type_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'seedlot', 'stock_type')->cvterm_id(); +my $stock_type_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession', 'stock_type')->cvterm_id(); + +print STDERR "Seedlot type id = $seedlot_type_id\n"; + +open(my $F, "<", $opt_i) || die "Can't open file $opt_i"; + + +while (<$F>) { + + chomp($_); + + my ($empty, $seedlot, @row) = split /\t/, $_; + + print STDERR "Reading row '$empty', $seedlot, and the rest is: ". join(",", @row)."\n"; + + my $seedlot_row = $schema->resultset("Stock::Stock")->find( { uniquename => $seedlot, type_id=> $seedlot_type_id }); + + my $name = ""; + my $type = ""; + + if (! defined($seedlot_row)) { + $name = "[SEEDLOT NOT IN DB]"; + print STDERR "ROW === ".Dumper($seedlot_row); + } + else { + + print STDERR "FOUND SEEDLOT!\n"; + + my $seedlot = CXGN::Stock::Seedlot->new( schema => $schema, seedlot_id => $seedlot_row->stock_id() ); + + if (my $accession = $seedlot->accession()) { + $name = $accession->[1]; + $type = "accession"; + } + elsif (my $cross = $seedlot->cross()) { + $name = $cross->[1]; + $type = "cross"; + } + + } + print join("\t", $name, $seedlot, @row)."\n"; +} + +close($F); +print STDERR "Script Complete.\n"; diff --git a/bin/fix_tissue_samples_plot_associations.pl b/bin/fix_tissue_samples_plot_associations.pl new file mode 100644 index 0000000000..190a2947a4 --- /dev/null +++ b/bin/fix_tissue_samples_plot_associations.pl @@ -0,0 +1,142 @@ + +=head1 NAME + +fix_tissue_samples_plot_associations.pl - a script to fix associations between tissue samples and plots, including plate wells + +=head1 DESCRIPTION + +perl bin/fix_tissue_samples_plot_associations.pl -H -D file.tsv + +where file.tsv is a tab delimited file with the following two columns: + +=over 16 + +=item tissue_sample + +for the tissue sample uniquename + +=item plot_name + +for the plot name to be associated with the tissue_sample + +=back + +Checks that the tissue_sample and plot are of the correct type and not already associated with each other. +Emits an error to STDERR and proceeds to the next entry if conditions are not met. + +=head1 AUTHOR + +Lukas Mueller + +=head1 DATE + +June 2025 + +=cut + +use strict; + +use Getopt::Std; +use CXGN::DB::InsertDBH; +use CXGN::DB::Schemas; +use CXGN::Stock; +use SGN::Model::Cvterm; + +our($opt_H, $opt_D, $opt_P); +getopts('H:D:xP:'); + +my $pw = $opt_P; + +if (! $pw) { + print "Password for $opt_H / $opt_D: \n"; + $pw = (); + chomp($pw); +} + +my $dsn = 'dbi:Pg:database='.$opt_D.";host=".$opt_H.";port=5432"; + +my $dbh = DBI->connect($dsn, "postgres", $pw, { AutoCommit => 0, RaiseError=>1 }); + +print STDERR "Connecting to DBI schema...\n"; +my $bcs_schema = Bio::Chado::Schema->connect($dsn, "postgres", $pw); + +my $tissue_sample_of_id = SGN::Model::Cvterm->get_cvterm_row($bcs_schema, 'tissue_sample_of', 'stock_relationship')->cvterm_id(); + +my $s = CXGN::DB::Schemas->new({ dbh => $dbh }); +my $schema = $s->bcs_schema(); +my $file = shift; + +open(my $F, "<", $file) || die "Can't open file $file\n"; + +while (<$F>) { + chomp; + + my ($tissue_sample_name, $plot_name) = split /\t/; + + print STDERR "Dealing with $plot_name and $tissue_sample_name...\n"; + my $q = "SELECT plot.uniquename, plot.stock_id, stock_relationship.stock_relationship_id, plot_type.name, tissue_sample.uniquename, tissue_sample.stock_id, tissue_sample_type.name FROM stock as plot join stock_relationship on(plot.stock_id = stock_relationship.object_id) join stock as tissue_sample on(stock_relationship.subject_id=tissue_sample.stock_id) join cvterm as plot_type on(plot.type_id=plot_type.cvterm_id) join cvterm as tissue_sample_type on(tissue_sample.type_id=tissue_sample_type.cvterm_id) where plot_type.name='plot' and tissue_sample.uniquename=?"; + + my $h = $dbh->prepare($q); + + $h->execute($tissue_sample_name); + + if (my ($already_associated_plot_name, + $already_associated_plot_id, + $stock_rel_id, + $already_associated_plot_type, + $associated_tissue_sample_name, + $tissue_sample_id, + $tissue_sample_type) = $h->fetchrow_array()) { + + if ($already_associated_plot_name) { + print STDERR "PLOT $already_associated_plot_name already assigned to tissue sample $tissue_sample_name. Skipping. \n"; + next(); + } + elsif ($tissue_sample_type ne 'tissue_sample') { + print STDERR "TISSUE SAMPLE $tissue_sample_name IS NOT OF TYPE tissue_sample. Skipping.\n"; + } + + } + else { + print STDERR "Associating $plot_name with $tissue_sample_name...\n"; + + my $pq = "SELECT stock_id, cvterm.name FROM stock join cvterm on(stock.type_id=cvterm.cvterm_id) where uniquename = ?"; + my $h = $dbh->prepare($pq); + $h->execute($plot_name); + + my ($plot_id, $type) = $h->fetchrow_array(); + + if ($type ne 'plot') { + print STDERR "$plot_name IS OF TYPE $type and not TYPE PLOT! Skipping.\n"; + next(); + } + + my $tsq = "SELECT stock_id, cvterm.name FROM stock join cvterm on(stock.type_id=cvterm.cvterm_id) where uniquename = ?"; + my $h = $dbh->prepare($tsq); + $h->execute($tissue_sample_name); + + my ($tissue_sample_id, $type) = $h->fetchrow_array(); + + if ($type ne 'tissue_sample') { + print STDERR "$tissue_sample_name is not of type tissue_sample, instead it is $type. Skipping.\n"; + next(); + } + + + my $iq = "INSERT INTO stock_relationship (object_id, subject_id, type_id) values (? , ?, ?)"; + + my $ih = $dbh->prepare($iq); + $ih->execute($plot_id, $tissue_sample_id, $tissue_sample_of_id); + print STDERR "$iq with $plot_id, $tissue_sample_id, $tissue_sample_of_id\n"; + + } +} + +print STDERR "COMMITTING...\n"; +$dbh->commit(); + + +print STDERR "Done.\n"; + + + diff --git a/bin/load_composed_cvprops.pl b/bin/load_composed_cvprops.pl index a919b9c53e..271af0844a 100755 --- a/bin/load_composed_cvprops.pl +++ b/bin/load_composed_cvprops.pl @@ -7,6 +7,7 @@ =head1 SYNOPSIS load_composed_cvprops.pl -H [dbhost] -D [dbname] -T [trait_ontology cv name] -c [composed_trait_ontology cv name] -o [object_ontology cv name] -a [attribute_ontology cv name] -m [method_ontology cv name] -u [unit_ontology cv name] -t [time_ontology cv name] +-d [metadata_ontology] =head1 COMMAND-LINE OPTIONS @@ -14,13 +15,14 @@ =head1 COMMAND-LINE OPTIONS -D database name optional: - -T [trait_ontology cv name] - -c [composed_trait_ontology cv name] - -o [object_ontology cv name] - -a [attribute_ontology cv name] + -T [trait_ontology cv name] cassava_trait - the main trait ontology for the database + -c [composed_trait_ontology cv name] composed_trait + -o [object_ontology cv name] cxgn_plant_section | cxgn_plant_level_ontology + -a [attribute_ontology cv name] cxgn_plant_treatment | cxgn_plant_cycle -m [method_ontology cv name] - -u [unit_ontology cv name] - -t [time_ontology cv name] + -u [unit_ontology cv name] cxgn_units_ontology + -t [time_ontology cv name] cxgn_time_ontology + -d [metadata_ontology cv name] cxgn_metadata =head2 DESCRIPTION @@ -42,12 +44,12 @@ =head2 AUTHOR use CXGN::DB::Connection; use Try::Tiny; -our ( $opt_H, $opt_D, $opt_T, $opt_c, $opt_o, $opt_a, $opt_m, $opt_u, $opt_t ); -getopts('H:D:T:c:o:a:m:u:t:'); +our ( $opt_H, $opt_D, $opt_T, $opt_c, $opt_o, $opt_a, $opt_m, $opt_u, $opt_t, $opt_d ); +getopts('H:D:T:c:o:a:m:u:t:d:'); sub print_help { print STDERR -"A script to load composed cvprops\nUsage: load_composed_cvprops.pl -H [dbhost] -D [dbname] -T [trait_ontology cv name] -c [composed_trait_ontology cv name] -o [object_ontology cv name] -a [attribute_ontology cv name] -m [method_ontology cv name] -u [unit_ontology cv name] -t [time_ontology cv name] \n"; +"A script to load composed cvprops\nUsage: load_composed_cvprops.pl -H [dbhost] -D [dbname] -T [trait_ontology cv name] -c [composed_trait_ontology cv name] -o [object_ontology cv name] -a [attribute_ontology cv name] -m [method_ontology cv name] -u [unit_ontology cv name] -t [time_ontology cv name] -d [metadata_ontology cv name]\n"; } if ( !$opt_D || !$opt_H ) { @@ -75,7 +77,8 @@ sub print_help { attribute_ontology => $opt_a, method_ontology => $opt_m, unit_ontology => $opt_u, - time_ontology => $opt_t + time_ontology => $opt_t, + metadata_ontology => $opt_d ); my $coderef = sub { @@ -96,7 +99,7 @@ sub print_help { if ( !$ontology ) { print STDERR -"No cv was found with the name '$value' in database '$opt_D'.\n"; +"No cv was found with the name '$value' in database '$opt_D'. Make sure all db patches were run\n"; } $ontology_cvtype = $schema->resultset("Cv::Cvterm")->find( diff --git a/bin/load_genotypes_vcf_cxgn_postgres.pl b/bin/load_genotypes_vcf_cxgn_postgres.pl index ebd0294416..ca2941425c 100644 --- a/bin/load_genotypes_vcf_cxgn_postgres.pl +++ b/bin/load_genotypes_vcf_cxgn_postgres.pl @@ -68,6 +68,7 @@ =head1 AUTHOR =cut use strict; +use warnings; use Getopt::Std; use Data::Dumper; @@ -171,7 +172,7 @@ =head1 AUTHOR open (my $Fout, ">", $opt_o) || die "Can't open file $opt_o\n"; open (my $F, "<", $file) or die "Can't open file $file \n"; my @outline; - my $lastcol; + my $lastcol = 0; while (<$F>) { if ($_ =~ m/^\##/) { print $Fout $_; @@ -245,7 +246,7 @@ =head1 AUTHOR my $organism_q = "SELECT organism_id FROM organism WHERE species = ?"; my @found_organisms; -my $h = $schema->storage->dbh()->prepare($organism_q); +$h = $schema->storage->dbh()->prepare($organism_q); $h->execute($organism_species); while (my ($organism_id) = $h->fetchrow_array()){ push @found_organisms, $organism_id; @@ -270,7 +271,12 @@ =head1 AUTHOR }); $parser->load_plugin($opt_c); -$parser->parse_with_iterator(); +my $parser_return = $parser->parse_with_iterator(); +if ($parser->get_parse_errors()) { + my $parse_errors = $parser->get_parse_errors(); + print STDERR Dumper $parse_errors; + die("parse errors"); +} my $project_id; my $protocol = $parser->protocol_data(); @@ -278,7 +284,6 @@ =head1 AUTHOR $protocol->{'reference_genome_name'} = $reference_genome_name; $protocol->{'species_name'} = $organism_species; - my $vcf_genotyping_type = $opt_T ? $opt_T : 'vcf_snp_genotyping'; # my $genotyping_type; my $genotype_data_type; @@ -337,7 +342,7 @@ =head1 AUTHOR die; } if (scalar(@{$verified_errors->{warning_messages}}) > 0){ - my $warning_string = join ', ', @{$verified_errors->{warning_messages}}; + my $warning_string = join "\n", @{$verified_errors->{warning_messages}}; if (!$opt_A){ print STDERR Dumper $warning_string; print STDERR "You can accept these warnings and continue with store if you use -A\n"; @@ -345,6 +350,53 @@ =head1 AUTHOR } } + my @protocol_match_errors; + if ($protocol_id) { + my $new_marker_data = $protocol->{markers}; + my $stored_protocol = CXGN::Genotype::Protocol->new({ + bcs_schema => $schema, + nd_protocol_id => $protocol_id + }); + my $stored_markers = $stored_protocol->markers(); + + my @all_stored_markers = keys %$stored_markers; + my %compare_marker_names = map {$_ => 1} @all_stored_markers; + my @mismatch_marker_names; + while (my ($chrom, $new_marker_data_1) = each %$new_marker_data) { + while (my ($marker_name, $new_marker_details) = each %$new_marker_data_1) { + if (exists($compare_marker_names{$marker_name})) { + while (my ($key, $value) = each %$new_marker_details) { + if ($value ne ($stored_markers->{$marker_name}->{$key})) { + push @protocol_match_errors, "Marker $marker_name in your file has $value for $key, but in the previously stored protocol shows ".$stored_markers->{$marker_name}->{$key}; + } + } + } else { + push @mismatch_marker_names, $marker_name; + } + } + } + + if (scalar(@mismatch_marker_names) > 0){ + foreach my $error ( sort @mismatch_marker_names) { + print STDERR "$error\n"; + } + print STDERR "These marker names in your file are not in the selected protocol.\n"; + die; + } + + if (scalar(@protocol_match_errors) > 0){ + my $protocol_warning; + foreach my $match_error (@protocol_match_errors) { + $protocol_warning .= "$match_error\n"; + } + if (!$opt_A){ + print STDERR Dumper $protocol_warning; + print STDERR "Protocol match error\n"; + die; + } + } + } + $store_genotypes->store_metadata(); my $result = $store_genotypes->store_identifiers(); $protocol_id = $result->{nd_protocol_id}; diff --git a/bin/load_genotyping_plates.pl b/bin/load_genotyping_plates.pl index 0fedd8af5b..0b39df8ce5 100644 --- a/bin/load_genotyping_plates.pl +++ b/bin/load_genotyping_plates.pl @@ -5,7 +5,9 @@ =head1 SYNOPSIS -load_genotyping_plates.pl -H [dbhost] -D [dbname] -i inFile -b [breeding program name] -u [username] -l location [-t] +NOTE: You need to create the genotyping project in the database first. With the -g option, provide the name of genotyping project the plates should be associated with. Metadata such as year, location and breeding program will be loaded from the genotyping object directly. + +load_genotyping_plates.pl -H [dbhost] -D [dbname] -i inFile -u [username] -g genotyping_project [-t] -f format =head1 COMMAND-LINE OPTIONS @@ -27,21 +29,13 @@ =head1 COMMAND-LINE OPTIONS username (must be in the database) -=item -b - -breeding program name (must be in the database) - =item -t Test run . Rolling back at the end. -=item -l - -location +=item -g -=item -y - -year +genotyping project name (the genotyping project to which this plate is associated) =back @@ -49,37 +43,7 @@ =head2 DESCRIPTION Load genotyping plate layouts for many plates -Minimal metadata requirements are - -=over 3 - -=item - -trial_name - -=item - -trial_description (can also be built from the trial name, type, year, location) - -=item - -trial_type (read from an input file) - -=item - -trial_location geo_description ( must be in the database - nd_geolocation.description - can be read from metadata file) - -=item - -year (can be read from the metadata file ) - -=item - -breeding_program (provide with option -b ) - -=back - -The infile is an Excel file (.xls format) with the following columns: +The infile is a tab delimited file with the following columns: =over 3 @@ -137,12 +101,10 @@ =head2 AUTHORS use CXGN::Trial::TrialCreate; -my ( $help, $dbhost, $dbname, $infile, $sites, $types, $test, $username, $breeding_program_name, $metadata_file, $location, $year, $format ); +my ( $help, $dbhost, $dbname, $infile, $sites, $types, $test, $username, $genotyping_project, $format ); GetOptions( 'i=s' => \$infile, - 'b=s' => \$breeding_program_name, - 'l=s' => \$location, - 'y=s' => \$year, + 'g=s' => \$genotyping_project, 't' => \$test, 'f=s' => \$format, 'user|u=s' => \$username, @@ -154,7 +116,7 @@ =head2 AUTHORS pod2usage(1) if $help; -if (!$infile || !$breeding_program_name || !$username || !$dbname || !$dbhost ) { +if (!$infile || !$username || !$dbname || !$dbhost ) { pod2usage( { -msg => 'Error. Missing options!' , -verbose => 1, -exitval => 1 } ) ; } @@ -173,23 +135,20 @@ =head2 AUTHORS my $phenome_schema = CXGN::Phenome::Schema->connect( sub { $dbh->get_actual_dbh() } , {on_connect_do => ['SET search_path TO phenome;'] } ); -# Breeding program for associating the trial/s ## +# check if genotyping project exists # - -my $breeding_program = $schema->resultset("Project::Project")->find( - { - 'me.name' => $breeding_program_name, - 'type.name' => 'breeding_program', - }, +my $genotyping_project_row = $schema->resultset("Project::Project")->find( { - join => { projectprops => 'type' } , - } ) ; + 'name' => $genotyping_project, +# 'type.name' => 'genotyping_project_name', + } ); -if (!$breeding_program) { die "Breeding program $breeding_program_name does not exist in the database. Check your input \n"; } -print "Found breeding program $breeding_program_name " . $breeding_program->project_id . "\n"; +if (! $genotyping_project_row) { die "Please enter a valid genotyping project. You may have to create it before running this script."; } +my $genotyping_project_id = $genotyping_project_row->project_id(); + if (!$format) { - die "Please specify format (-f) as CIP. No other format is supported right now\n"; + $format = "standard"; } my $sp_person_id= CXGN::People::Person->get_person_by_username($dbh, $username); @@ -220,22 +179,14 @@ =head2 AUTHORS my %phen_params = map { if ($_ =~ m/^\w+\|(\w+:\d{7})$/ ) { $_ => $1 } } @trial_columns ; delete $phen_params{''}; -my @traits = (keys %phen_params) ; -print "Found traits " . Dumper(\%phen_params) . "\n" ; -#foreach my $trait_string ( keys %phen_params ) { -# my ($trait_name, $trait_accession) = split "|", $col_header ; -# my ($db_name, $dbxref_accession) = split ":" , $trait_accession ; -#} - - my %trial_design_hash; #multi-level hash of hashes of hashrefs my %phen_data_by_trial; # -#plot_name accession_name plot_number block_number trial_name trial_description trial_location year trial_type is_a_control rep_number range_number row_number col_number - - # CIP format: -## Item Plate ID Intertek plate/well ID CIP Number Breeder ID +# Item Plate ID Intertek plate/well ID CIP Number Breeder ID + +# standard format: +# Item Plate ID Intertek plate/well ID accession name Breeder ID my $operator; @@ -280,37 +231,36 @@ =head2 AUTHORS $trial_design_hash{$trial_name}{$plot_number}->{row_number} = $row_number; $trial_design_hash{$trial_name}{$plot_number}->{col_number} = $col_number; } - else { - $accession = $spreadsheet->value_at($plot_name, "accession_name"); - $plot_number = $spreadsheet->value_at($plot_name, "plot_number"); - $block_number = $spreadsheet->value_at($plot_name, "block_number"); - $trial_name = $spreadsheet->value_at($plot_name, "trial_name"); - $is_a_control = $spreadsheet->value_at($plot_name, "is_a_control"); - $rep_number = $spreadsheet->value_at($plot_name, "rep_number"); - $range_number = $spreadsheet->value_at($plot_name, "range_number"); - $row_number = $spreadsheet->value_at($plot_name, "row_number"); - $col_number = $spreadsheet->value_at($plot_name, "col_number"); - - if (!$plot_number) { - $plot_number = 1; - use List::Util qw(max); - my @keys = (keys %{ $trial_design_hash{$trial_name} } ); - my $max = max( @keys ); - if ( $max ) { - $max++; - $plot_number = $max ; - } + else { + + $accession = $spreadsheet->value_at($plot_name, "accession name"); + $plot_number = $spreadsheet->value_at($plot_name, "Intertek plate/well ID"); + $trial_name = $spreadsheet->value_at($plot_name, "Plate ID"); + $operator = $spreadsheet->value_at($plot_name, "Breeder ID"); + + if (! $accession) { + print STDERR "Ignoring entry for plot_number $plot_number as accession is empty - presumably a check?\n"; + next; + } # some plates have empty wells - ignore + + if ($plot_number =~ m/^([A-Ha-h])(\d+)$/) { + $row_number = $1; + $col_number = $2; + } + + $is_a_control = 0; + if ($accession eq "") { + $is_a_control = 1; } + if (! $row_number ) { die "Weird well number: $plot_number\n"; } + $trial_design_hash{$trial_name}{$plot_number}->{plot_number} = $plot_number; $trial_design_hash{$trial_name}{$plot_number}->{stock_name} = $accession; $trial_design_hash{$trial_name}{$plot_number}->{plot_name} = $plot_name; - $trial_design_hash{$trial_name}{$plot_number}->{block_number} = $block_number; - $trial_design_hash{$trial_name}{$plot_number}->{rep_number} = $rep_number; - $trial_design_hash{$trial_name}{$plot_number}->{is_a_control} = $is_a_control; - $trial_design_hash{$trial_name}{$plot_number}->{range_number} = $range_number; $trial_design_hash{$trial_name}{$plot_number}->{row_number} = $row_number; $trial_design_hash{$trial_name}{$plot_number}->{col_number} = $col_number; + } # Add the plot name into the multi trial data hashref of hashes @@ -337,27 +287,42 @@ =head2 AUTHORS ####### my $coderef= sub { + + my $trial = CXGN::Trial->new( { bcs_schema => $schema, trial_id => $genotyping_project_id }); + my $location_data = $trial->get_location(); + my $location_name = $location_data->[1]; + my $description = $trial->get_description(); + my $genotyping_facility = $trial->get_genotyping_facility(); + my $plate_year = $trial->get_year(); + + my $program_object = CXGN::BreedersToolbox::Projects->new( { schema => $schema }); + my $breeding_program_data = $program_object->get_breeding_programs_by_trial($genotyping_project_id); + my $breeding_program_name = $breeding_program_data->[0]->[1]; + + print STDERR "Working with genotyping project name $genotyping_project\n"; foreach my $trial_name (keys %multi_trial_data ) { - my $trial_create = CXGN::Trial::TrialCreate->new({ - chado_schema => $schema, - dbh => $dbh, - design_type => 'genotyping_plate', - design => $trial_design_hash{$trial_name}, - program => $breeding_program->name(), - trial_year => $year, - trial_description => $trial_name, - trial_location => $location, - trial_name => $trial_name, - operator => $operator, - owner_id => $sp_person_id, - is_genotyping => 1, - genotyping_user_id => $sp_person_id, - genotyping_plate_format => 96, - genotyping_plate_sample_type => 'accession', - - - }); + my $trial_create = CXGN::Trial::TrialCreate->new( + { + chado_schema => $schema, + dbh => $dbh, + design_type => 'genotyping_plate', + design => $trial_design_hash{$trial_name}, + program => $breeding_program_name, + trial_year => $plate_year, + trial_description => $description, + trial_location => $location_name, + trial_name => $trial_name, + operator => $operator, + owner_id => $sp_person_id, + is_genotyping => 1, + genotyping_user_id => $sp_person_id, + genotyping_plate_format => $format, + genotyping_plate_sample_type => 'accession', + genotyping_project_id => $genotyping_project_id, + genotyping_facility => $genotyping_facility, + }); + try { $trial_create->save_trial(); } catch { @@ -370,12 +335,6 @@ =head2 AUTHORS $schema->txn_do($coderef); if (!$test) { print "Transaction succeeded! Commiting project and its metadata \n\n"; } } catch { - # Transaction failed -# foreach my $value ( sort keys %seq ) { -# my $maxval= $seq{$value} || 0; -# if ($maxval) { $dbh->do("SELECT setval ('$value', $maxval, true)") ; } -# else { $dbh->do("SELECT setval ('$value', 1, false)"); } -# } die "An error occured! Rolling back and reseting database sequences!" . $_ . "\n"; }; diff --git a/bin/load_images.pl b/bin/load_images.pl new file mode 100644 index 0000000000..50418081c0 --- /dev/null +++ b/bin/load_images.pl @@ -0,0 +1,467 @@ +#!/usr/bin/perl + +=head1 NAME + +load_images.pl + +=head1 SYNOPSYS + +load_images.pl -D database_name -H hostname -i dirname -r chado table name [script will load image ids into ChadoTableprop ] + +=head1 DESCRIPTION + +Loads images into the SGN database, using the SGN::Image framework. +Then link the loaded image with the user-supplied chado objects (e.g. stock, nd_experiment) + +Requires the following parameters: + +=over 8 + +=item -D + +database name + +=item -H + +host name + +=item -m + +map file. If provided, creates links between stocks and images. +Row labels are expected to be unique file names, column header for the associated stocks +are 'file name', 'name', and 'cvterm'. The cvterm column is optional. It should +contain the name of the cvterm, not the id. + +=item -i + +a dirname that contains image filenames or subdirectories named after database accessions, containing one or more images (see option -d) . + +=item -u + +use name - from sgn_people.sp_person. + +=item -b + +the dir where the database stores the images (the concatenated values from image_path and image_dir from sgn_local.conf or sgn.conf) + +=item -d + +files are stored in sub directories named after database accessions + +=item -e + +image file extension. Defaults to 'jpg' + +=item -t + +trial mode . Nothing will be stored. + +=item -P + +password (for unit tests) + +=item -y + +confirm automatically (for unit tests) + +=back + +Errors and messages are output on STDERR. + +=head1 AUTHOR(S) + +Naama Menda (nm249@cornell.edu) October 2010. + +Tweaks and move to sgn/bin: Lukas Mueller (lam87@cornell.edu) December 2023. + +=cut + +use strict; + +use CXGN::Metadata::Schema; +use CXGN::Metadata::Metadbdata; +use CXGN::DB::InsertDBH; +use CXGN::Image; +use Bio::Chado::Schema; +use CXGN::People::Person; +use Carp qw /croak/; +use Data::Dumper qw / Dumper /; + +use File::Basename; +use SGN::Context; +use Getopt::Std; + +use CXGN::Tools::File::Spreadsheet; +use File::Glob qw | bsd_glob |; + +our ($opt_H, $opt_D, $opt_t, $opt_i, $opt_u, $opt_r, $opt_d, $opt_e, $opt_m, $opt_b, $opt_P, $opt_y); +getopts('H:D:u:i:e:f:tdr:m:b:P:y'); + +my $dbhost = $opt_H; +my $dbname = $opt_D; +my $dbpass = $opt_P; +my $dirname = $opt_i; +my $sp_person=$opt_u; +my $db_image_dir = $opt_b; +my $chado_table = $opt_r; +my $ext = $opt_e || 'jpg'; + +print STDERR "LOAD IMAGES STARTING... $dbhost $dbname $dbpass $dirname\n"; + +if (!$dbhost && !$dbname) { + print STDERR "dbhost = $dbhost , dbname = $dbname\n"; + print STDERR "opt_t = $opt_t, opt_u = $opt_u, opt_r = $chado_table, opt_i = $dirname\n"; + usage(); +} + +if (!$dirname) { print STDERR "dirname = $dirname\n" ; usage(); } + +my $dbh; + +if ($opt_P) { + print STDERR "PASSWORD SUPPLIED... CONNECTING...\n"; + $dbh = CXGN::DB::Connection->new( { dbhost=> $dbhost, dbname=> $dbname, dbpass => $opt_P, dbuser => "postgres" }); +} +else { + print STDERR "INTERACTIVE PASSWORD...\n"; + $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost, dbname=>$dbname } ); +} + + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } , { on_connect_do => ['SET search_path TO public;'] } ); + +print STDERR "Generate metadata_id... "; +my $metadata_schema = CXGN::Metadata::Schema->connect("dbi:Pg:database=$dbname;host=".$dbh->dbhost(), "postgres", $dbh->dbpass(), {on_connect_do => "SET search_path TO 'metadata', 'public'", }); + +my $sp_person_id= CXGN::People::Person->get_person_by_username($dbh, $sp_person); +my %name2id = (); + + +#my $ch = SGN::Context->new(); +if (! $opt_y) { + print "PLEASE VERIFY:\n"; + print "Using dbhost: $dbhost. DB name: $dbname. \n"; + print "Path to image is: $db_image_dir\n"; + print "CONTINUE? "; + my $a = (); + if ($a !~ /[yY]/) { exit(); } +} + +my %image_hash = (); # used to retrieve images that are already loaded +my %connections = (); # keep track of object -- image connections that have already been made. + +print STDERR "Caching stock table...\n"; +my $object_rs = $schema->resultset("Stock::Stock")->search( { } ) ; +while (my $object = $object_rs->next ) { + my $id = $object->stock_id; + my $name = $object->uniquename; + $name2id{lc($name)} = $id; +} + +# cache image chado object - image links to prevent reloading of the +# same data +# +print "Caching image $chado_table links...\n"; + +my $q = "SELECT * FROM phenome.stock_image"; +my $sth = $dbh->prepare($q); +$sth->execute(); +while ( my $hashref = $sth->fetchrow_hashref() ) { + my $image_id = $hashref->{image_id}; + my $chado_table_id = $hashref->{stock_id}; ##### table specific + + if ($chado_table_id % 10000 == 0) { + print STDERR "CACHING $chado_table_id\n"; + } + + my $i = CXGN::Image->new(dbh=>$dbh, image_id=>$image_id, image_dir=>$db_image_dir); # SGN::Image...$ch + my $original_filename = $i->get_original_filename(); + $image_hash{$original_filename} = $i; # this doesn't have the file extension + $connections{$image_id."-".$chado_table_id}++; +} + +#open (ERR, ">load_bcs_images.err") || die "Can't open error file\n"; + +my @files; +if (! $opt_d) { + @files = bsd_glob "$dirname/*.$ext"; +} +else { + @files = bsd_glob "$dirname/*" if $opt_d ; +} + +print STDERR "DIRS = ".(join("\n", @files))."\n"; + +my @sub_files; + +my $new_image_count = 0; + +my $metadata = CXGN::Metadata::Metadbdata->new($metadata_schema, $sp_person); +my $metadata_id = $metadata->store()->get_metadata_id(); + +#read from spreadsheet: +my $map_file = $opt_m; # +my %name_map; + +if ($opt_m) { + eval { + my $s = CXGN::Tools::File::Spreadsheet->new($map_file); # + my @rows = $s->row_labels(); # + my @cols = $s->column_labels(); + my $image_id; + foreach my $file_name (@rows) { # + my $stock_name = $s->value_at($file_name, 'name'); # + print STDERR "RETRIEVED STOCK NAME: $stock_name\n"; + my $cvterm_name; + if (grep(/cvterm/, @cols)) { + $cvterm_name = $s->value_at($file_name, 'cvterm'); + print "RETRIEVED CVTERM: $cvterm_name\n"; + } + + $name_map{$file_name} = $stock_name; + if (my $image_id = store_image($dbh, $db_image_dir, \%image_hash, \%name2id, $chado_table, $stock_name, $opt_i."/".$file_name, "", $sp_person_id, $new_image_count)) { + $new_image_count++; + + my $stock_row = $schema->resultset("Stock::Stock")->find( { uniquename => $stock_name } ); + if (!$stock_row) { + print STDERR "STOCK $stock_name NOT FOUND! PLEASE CHECK THIS ENTRY! IMAGE STORED WITHOUT LINKING.\n"; + } + else { + print STDERR "FOUND STOCK $stock_name... associating...\n"; + link_image($image_id, $stock_row->stock_id, $metadata_id); + + if ($cvterm_name) { + print STDERR "LINKING IMAGE $image_id WITH cvterm $cvterm_name...\n"; + link_cvterm($image_id, $cvterm_name); + } + + } + + + + } + } + }; + if ($@) { + print STDERR "ERROR OCCURRED WHILE SAVING NEW INFORMATION. $@\n"; + $dbh->rollback(); + } + else { + $dbh->commit(); + } + + print STDERR "DONE WITH MAPPING FILE $opt_m\n"; + exit(0); +} + +print STDERR "Starting to process ".scalar(@files)." images...\n"; + +foreach my $file (@files) { + eval { + chomp($file); + @sub_files = ($file); + @sub_files = bsd_glob "$file/*"; # if $opt_d; + + print STDERR "FILES FOR $file: ".Dumper(\@sub_files)."\n"; + + my $object = basename($file, ".$ext" ); + +# if (!$plot) { die "File $file has no object name in it!"; } + my $stock = $schema->resultset("Stock::Stock")->find( { + stock_id => $name2id{ lc($object) } } ); + foreach my $filename (@sub_files) { + + chomp $filename; + + print STDERR "FILENAME NOW: $filename\n"; + my $image_base = basename($filename); + my ($object_name, $description, $extension); + if ($opt_m) { + $object_name = $name_map{$object . "." . $ext } ; + } + + print STDERR "OBJECT = $object...\n"; +# if ($image_base =~ /(.*?)\_(.*?)(\..*?)?$/) { + if ($image_base =~ m/(.*)(\.$ext)/i) { + $extension = $2; + $image_base = $1; + } + if ($image_base =~ m/(.*)\_(.*)/) { + $object_name = $1; + $description = $2; + + } + else { + $object_name = $image_base; + } + print STDERR "Object: $object OBJECT NAME: $object_name DESCRPTION: $description EXTENSIO: $extension\n"; + + + print STDOUT "Processing file $file...\n"; + print STDOUT "Loading $object_name, image $filename\n"; + print STDERR "Loading $object_name, image $filename\n"; + my $image_id; # this will be set later, depending if the image is new or not + if (! -e $filename) { + warn "The specified file $filename does not exist! Skipping...\n"; + next(); + } + + if (!exists($name2id{lc($object)})) { + message ("$object does not exist in the database...\n"); + } + + else { + print STDERR "Adding $filename...\n"; + if (exists($image_hash{$filename})) { + print STDERR "$filename is already loaded into the database...\n"; + $image_id = $image_hash{$filename}->get_image_id(); + $connections{$image_id."-".$name2id{lc($object)}}++; + if ($connections{$image_id."-".$name2id{lc($object)}} > 1) { + print STDERR "The connection between $object and image $filename has already been made. Skipping...\n"; + } + elsif ($image_hash{$filename}) { + print STDERR qq { Associating $chado_table $name2id{lc($object)} with already loaded image $filename...\n }; + } + } + else { + print STDERR qq { Generating new image object for image $filename and associating it with $chado_table $object, id $name2id{lc($object) } ...\n }; + + if ($opt_t) { + print STDOUT qq { Would associate file $filename to $chado_table $object_name, id $name2id{lc($object)}\n }; + $new_image_count++; + } + else { + # my $image = CXGN::Image->new(dbh=>$dbh, image_dir=>$db_image_dir); + # $image_hash{$filename}=$image; + + # my $error; + # ($image_id, $error) = $image->process_image("$filename", $chado_table , $name2id{lc($object)}, 1); + + # print STDERR "IMAGE ID $image_id, ERROR: $error\n"; + + # if ($error eq "ok") { + # $image->set_description("$description"); + # $image->set_name(basename($filename , ".$ext")); + # $image->set_sp_person_id($sp_person_id); + # $image->set_obsolete("f"); + # $image_id = $image->store(); + # #link the image with the BCS object + # $new_image_count++; + # my $image_subpath = $image->image_subpath(); + # print STDERR "FINAL IMAGE PATH = $db_image_dir/$image_subpath\n"; + #} + + if ($image_id = store_image($dbh, $db_image_dir, \%image_hash, \%name2id, $chado_table, $object, $filename, $description, $sp_person_id, $new_image_count)) { + $new_image_count++; + } + } + } + if ($image_id) { link_image($image_id, $name2id{lc($object)}, $metadata_id); } + } + + + + # print STDERR "Connecting image $filename and id $image_id with stock ".$stock->stock_id()."\n"; + # #store the image_id - stock_id link + # my $q = "INSERT INTO phenome.stock_image (stock_id, image_id, metadata_id) VALUES (?,?,?)"; + # my $sth = $dbh->prepare($q); + # $sth->execute($stock->stock_id, $image_id, $metadata_id); + } + }; + if ($@) { + print STDERR "ERROR OCCURRED WHILE SAVING NEW INFORMATION. $@\n"; + $dbh->rollback(); + } + else { + $dbh->commit(); + } +} + +close(F); + +print STDERR "Inserted $new_image_count images.\n"; +print STDERR "Done. \n"; + + +sub store_image { + my $dbh = shift; + my $db_image_dir = shift; + my $image_hash = shift; + my $name2id = shift; + my $chado_table = shift; + my $object = shift; + my $filename = shift; + my $description = shift; + my $sp_person_id = shift; + + my $new_image_count; + + my $image = CXGN::Image->new(dbh=>$dbh, image_dir=>$db_image_dir); + $image_hash->{$filename}=$image; + + my ($image_id, $error) = $image->process_image("$filename", $chado_table , $name2id->{lc($object)}, 1); + + print STDERR "IMAGE ID $image_id, ERROR: $error\n"; + + if ($error =~ /duplicate/i) { + return 0; + } + + if ($error eq "ok") { + print STDERR "Storing image... \n"; + $image->set_description("$description"); + $image->set_name(basename($filename , ".$ext")); + $image->set_sp_person_id($sp_person_id); + $image->set_obsolete("f"); + $image_id = $image->store(); + #link the image with the BCS object + $new_image_count++; + my $image_subpath = $image->image_subpath(); + print STDERR "FINAL IMAGE PATH for IMAGE ".$image->get_image_id()." = $db_image_dir/$image_subpath\n"; + } + + return $image_id; +} + +sub link_image { + my $image_id = shift; + my $stock_id = shift; + my $metadata_id = shift; + print STDERR "Connecting image with id $image_id with stock ".$stock_id."\n"; + #store the image_id - stock_id link + my $q = "INSERT INTO phenome.stock_image (stock_id, image_id, metadata_id) VALUES (?,?,?)"; + my $sth = $dbh->prepare($q); + $sth->execute($stock_id, $image_id, $metadata_id); +} + +sub link_cvterm{ + my $image_id = shift; + my $cvterm_name = shift; + + my $q = "SELECT cvterm_id FROM cvterm where name = ? "; + my $h = $dbh->prepare($q); + + $h->execute($cvterm_name); + + my ($cvterm_id) = $h->fetchrow_array(); + + if ($cvterm_id && $image_id) { + my $iq = "INSERT INTO metadata.md_image_cvterm (image_id, cvterm_id) values (?, ?) RETURNING md_image_cvterm_id"; + my $ih = $dbh->prepare($iq); + $ih->execute($image_id, $cvterm_id); + + } + + +} + + +sub usage { + print "Usage: load_images.pl -D dbname [ cxgn | sandbox ] -H dbhost -t [trial mode ] -i input dir -r chado table name for the object to link with the image \n"; + exit(); +} + +sub message { + my $message=shift; + print STDERR $message; +} diff --git a/bin/load_marker_data.pl b/bin/load_marker_data.pl index 819b5b5d88..8c56a0891b 100644 --- a/bin/load_marker_data.pl +++ b/bin/load_marker_data.pl @@ -321,9 +321,9 @@ =head1 AUTHORS print STDERR "MARKER_ID = $marker_id\n"; - if (!$stock_id) { - print STDERR "The marker $marker_name exists, but is not on this map...\n"; next(); - } + # if (!$stock_id) { + # print STDERR "The marker $marker_name exists, but is not on this map...\n"; next(); + # } if (!$marker_id) { warn "marker $marker_name is not in the database!!!!\n"; diff --git a/bin/load_people.pl b/bin/load_people.pl index cc9de1dce0..35b7ab4c52 100644 --- a/bin/load_people.pl +++ b/bin/load_people.pl @@ -55,10 +55,11 @@ =head1 DESCRIPTION research_keywords research_interests webpage +password -The first four columns listed are required. +The first four columns listed are required. If no password is provided the system will create one. -The script outputs the username, first_name, last_name, email and assigned initial random password. This can be used to send the password to the user. +The script outputs the username, first_name, last_name, email and assigned initial random password. This can be used to send the password to the user. =head1 AUTHOR @@ -185,13 +186,31 @@ =head1 AUTHOR next(); } - $row = $people_schema->resultset("SpPerson")->find( { contact_email => $data{email} } ); - if ($row) { - print STDERR "Email $data{contact_email} already exists in the database. Skipping this row.\n"; - next(); + if ($data{email}) { + my $rs = $people_schema->resultset("SpPerson") + ->search( { '-or' => [ contact_email => $data{email}, private_email => $data{email}, pending_email => $data{email} ] } ); + + if ($rs->count > 0) { + print STDERR "Email $data{email} already exists in the database in contact_email, pending_email, or private_email field. Skipping this row.\n"; + next(); + } } - my $password =Crypt::RandPasswd->word( 8 , 8 ); + # $row = $people_schema->resultset("SpPerson")->find( { pending_email => $data{email} }); + # if ($row) { + # print STDERR "Email $data{email} already exists in the database in pending_email field. Skipping this row.\n"; + # next(); + # } + + my $password; + + if ($data{password}) { + $password = $data{password}; + } + else { + $password =Crypt::RandPasswd->word( 8 , 8 ); + } + if ($data{username}) { my $login = CXGN::People::Login->new($dbh); @@ -202,7 +221,6 @@ =head1 AUTHOR $login->set_password($password); print "$data{first_name}\t$data{last_name}\t$data{username}\t$data{email}\t$password\n"; - my $sp_person_id = $login->store(); diff --git a/bin/load_stocks.pl b/bin/load_stocks.pl index a85169a0eb..848ba718d0 100644 --- a/bin/load_stocks.pl +++ b/bin/load_stocks.pl @@ -10,13 +10,13 @@ =head1 SYNOPSIS =head1 COMMAND-LINE OPTIONS - -H host name - -D database name + -H host name + -D database name -i infile -u username for associating the new stocks -s species name - must be in the database. Can also be read from the input file -p population name - will create a new stock of type 'population' if doesn't exist. - -t Test run . Rolling back at the end. + -t Test run . Rolling back at the end. =head1 DESCRIPTION @@ -28,13 +28,14 @@ =head1 DESCRIPTION File format for infile (tab delimited): -accession genus species_name population_name synonyms other_stock_props ... +accession species_name population_name synonyms description other_stock_props ... Multiple synonyms can be specified, separated by the | symbol =head1 AUTHORS Naama Menda (nm249@cornell.edu) - April 2013 + Lukas Mueller (lam87@cornell.edu) - minor edits, November 2022 =cut @@ -57,20 +58,23 @@ =head1 AUTHORS use SGN::Model::Cvterm; use Getopt::Long; -my ( $dbhost, $dbname, $file, $population_name, $species, $username, $test ); +my ( $dbhost, $dbname, $file, $population_name, $species, $username, $password, $test ); GetOptions( 'i=s' => \$file, 'p=s' => \$population_name, 's=s' => \$species, 'u=s' => \$username, + 'P=s' => \$password, 't' => \$test, 'dbname|D=s' => \$dbname, 'dbhost|H=s' => \$dbhost, ); -my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost, +my $dbh = CXGN::DB::Connection->new( { dbhost=>$dbhost, dbname=>$dbname, + dbuser=>'postgres', + dbpass=>$password, dbargs => {AutoCommit => 1, RaiseError => 1} } @@ -111,8 +115,6 @@ =head1 AUTHORS print "Finding/creating cvterm for population\n"; my $population_cvterm = SGN::Model::Cvterm->get_cvterm_row($schema, 'population', 'stock_type'); - - # the cvterm for the accession # my $accession_cvterm = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession', 'stock_type'); @@ -129,7 +131,9 @@ =head1 AUTHORS my $syn_count; -# accession genus species population_name synonyms +print STDERR "COLUMN LABELS = ".join(", ", @columns)."\n"; + +# accession species population_name synonyms # my $coderef= sub { foreach my $accession (@rows ) { @@ -144,25 +148,33 @@ =head1 AUTHORS die "Species $species_name does not exist in the database! " if !$organism_id; } - my $population_name = $spreadsheet->value_at($accession, "population_name"); + my $population_names = $spreadsheet->value_at($accession, "population_name"); # new: can be more than one, | separated my $synonym_string = $spreadsheet->value_at($accession, "synonyms"); + my $description = $spreadsheet->value_at($accession, "description"); + my @synonyms = split /\|/ , $synonym_string; - print "Creating a stock for population $population_name (cvterm = " . $population_cvterm->name . ")\n"; - my $population = $stock_rs->find_or_create( - { - 'me.name' => $population_name, - 'me.uniquename' => $population_name, - 'me.organism_id' => $organism_id, - type_id => $population_cvterm->cvterm_id, - }, - { join => 'type' } - ); + my @population_rows; + if ($population_names) { + my @populations = split /\|/, $population_names; + + foreach my $name (@populations) { + print "Creating a stock for population $population_name (cvterm = " . $population_cvterm->name . ")\n"; + my $row = $stock_rs->find_or_create( { + 'me.name' => $name, + 'me.uniquename' => $name, + 'me.organism_id' => $organism_id, + type_id => $population_cvterm->cvterm_id, }, { join => 'type' } + ); + push @population_rows, $row; + } + } print "Find or create stock for accesssion $accession\n"; my $stock = $schema->resultset("Stock::Stock")->find_or_create( { organism_id => $organism_id, name => $accession, + description => $description, uniquename => $accession, type_id => $accession_cvterm->cvterm_id(), }); @@ -177,14 +189,19 @@ =head1 AUTHORS sp_person_id => $sp_person_id, }); - # the stock belongs to the population: - # add new stock_relationship + # the stock belongs to population(s): + # add new stock_relationship(s) # - print "Accession $accession is member_of population $population_name \n"; - $population->find_or_create_related('stock_relationship_objects', { - type_id => $member_of->cvterm_id(), - subject_id => $stock->stock_id(), - } ); + if ($population_names) { + foreach my $row (@population_rows) { + print "Accession $accession is member_of population ".$row->uniquename(); + $row->find_or_create_related('stock_relationship_objects', { + type_id => $member_of->cvterm_id(), + subject_id => $stock->stock_id(), + } ); + } + } + if ($synonym_string) {print "Adding synonyms #" . scalar(@synonyms) . "\n"; } foreach my $syn (@synonyms) { if ($syn && defined($syn) && ($syn ne $accession) ) { diff --git a/bin/load_trait_props.pl b/bin/load_trait_props.pl index 915304d38e..79d07d9382 100755 --- a/bin/load_trait_props.pl +++ b/bin/load_trait_props.pl @@ -28,18 +28,21 @@ =head2 DESCRIPTION trait_maximum trait_categories trait_details + trait_repeat_type trait_name: the name of the variable human readable form (e.g., "plant height in cm") trait_format: can be numeric, qualitative, date or boolean trait_default_value: is the value if no value is given trait_categories: are the different possible names of the categories, separated by /, for example "1/2/3/4/5" trait_details: string describing the trait categories + trait_repeat_type: one of 'single', 'multiple', 'time_series' =head2 AUTHOR -Jeremy D. Edwards (jde22@cornell.edu) + Jeremy D. Edwards (jde22@cornell.edu) - initial script, April 2014 + Lukas Mueller (lam87@cornell.edu) - added trait_repeat_type, Feb 2024 + -April 2014 =head2 TODO @@ -117,6 +120,7 @@ sub print_help { trait_maximum trait_categories trait_details + trait_repeat_type ); #check header for property names diff --git a/bin/merge_stocks.pl b/bin/merge_stocks.pl index 0815f63272..e3ba8381d3 100644 --- a/bin/merge_stocks.pl +++ b/bin/merge_stocks.pl @@ -12,6 +12,7 @@ =head1 DESCRIPTION -H the database host -D the database name -x flag; if present, delete the empty remaining accession + -P password mergefile.txt: A tab-separated file with two columns. Include the following header as the first line: bad name good name @@ -32,13 +33,16 @@ =head1 AUTHOR use CXGN::Stock; -our($opt_H, $opt_D, $opt_x); -getopts('H:D:x'); +our($opt_H, $opt_D, $opt_x, $opt_P); +getopts('H:D:xP:'); +my $pw = $opt_P; -print "Password for $opt_H / $opt_D: \n"; -my $pw = (); -chomp($pw); +if (! $pw) { + print "Password for $opt_H / $opt_D: \n"; + $pw = (); + chomp($pw); +} my $delete_merged_stock = $opt_x; diff --git a/bin/parse_vector.pl b/bin/parse_vector.pl new file mode 100644 index 0000000000..9a8f12312f --- /dev/null +++ b/bin/parse_vector.pl @@ -0,0 +1,34 @@ + +use strict; + +use Data::Dumper; +use CXGN::VectorViewer; + +my $file = shift; + +my $vv = CXGN::VectorViewer->new(); + +my $s = ""; +open(my $F,"<", $file) || die "Can't open file $file\n"; +while (<$F>) { + $s .= $_; +} + + +my $vector = $vv->parse_genbank($s); + +#my $ra = $vv->restriction_analysis('popular6bp'); + +print STDERR "\n\n\nPARSED OUTPUT: \n\n"; +#print STDERR Dumper($vv->sequence()); +print STDERR Dumper($vv->features()); +print STDERR Dumper($vv->re_sites()); +print STDERR Dumper($vv->metadata()); +print STDERR "Sequence length: ".length($vv->sequence)."\n"; + +print STDERR "DONE.\n"; + + + + + diff --git a/bin/remove_duplicate_samples_vcf.pl b/bin/remove_duplicate_samples_vcf.pl new file mode 100644 index 0000000000..0872acf741 --- /dev/null +++ b/bin/remove_duplicate_samples_vcf.pl @@ -0,0 +1,91 @@ +#!/usr/bin/perl + + +=head1 NAME + +remove_duplicate_samples_vcf.pl - renames and removes duplicate samples in a vcf file. +keeps one copy of the samples. + +=head1 DESCRIPTION + + perl remove_duplicate_samples_vcf.pl -i [vcf_input_file] -o [vcf_output_file] + + requires vcftools. Generates 3 files: a text file with the duplicated samples to remove, + a vcf files with the duplicates renamed and a vcf file with out the duplicates. + +=head1 AUTHOR + +Isaak Y Tecle + +=cut + +use strict; +use Getopt::Std; +use File::Slurp qw /read_file write_file/; + + +our($opt_i, $opt_o); + +getopts('i:o:'); + +open(my $V, "<", $opt_i) || die "Can't open vcf_file: $opt_i\n"; + +my $lines; +my @dupl_samples; + +while (<$V>) { + chomp(); + + if ($_ =~ m/^\#CHROM/) { + print STDERR "Parsing ids in vcf file...\n"; + my @orig_fields = split /\t/; + my @modified_fields; + for (my $i=0; $i <= $#orig_fields; $i++) { + my $field = $orig_fields[$i]; + if ($i < 9) { + print "\nkeeping the first 9 columns: $field\n"; + push @modified_fields, $field; + } else { + if (grep{$field eq $_} @modified_fields) { + $field = "${field}_dupl_${i}"; + print "$orig_fields[$i] at col $i is a duplicate -- modified its name to $field\n"; + push @dupl_samples, $field; + + } else { + print STDERR "\n$field at col $i is a unique sample\n"; + } + + push @modified_fields, $field; + } + } + + my $line = join("\t", @modified_fields); + $lines .= $line . "\n"; + } + else { + $lines .= $_ ."\n"; + } + +} + +my $dupl_samples = join("\n", @dupl_samples); +my $out_file = $opt_o =~ s/\.vcf//r; +my $remove_samples_file = "${out_file}_dupl_samples.txt"; +my $removed_vcf = "${out_file}_removed.vcf"; +my $renamed_vcf = "${out_file}_renamed.vcf"; + +print STDERR "Now writing to $remove_samples_file duplicate samples:\n$dupl_samples"; +write_file($remove_samples_file, $dupl_samples); + +print STDERR "Now writing to $renamed_vcf duplicate samples:\n$dupl_samples"; +write_file($renamed_vcf, $lines); + +print STDERR "Now removing duplicate samples:\n$dupl_samples"; +`vcftools --remove $remove_samples_file --vcf $renamed_vcf --recode --out $removed_vcf`; +my $recode_file = "${removed_vcf}.recode.vcf"; + +print STDERR "Renaming $recode_file to $removed_vcf\n"; +`mv $recode_file $removed_vcf`; +print STDERR"\nCleaned vcf file without the duplicates is $removed_vcf\n"; + +print STDERR "\nDone."; diff --git a/bin/rename_locations.pl b/bin/rename_locations.pl new file mode 100644 index 0000000000..8e8eddb23e --- /dev/null +++ b/bin/rename_locations.pl @@ -0,0 +1,109 @@ +#!/usr/bin/perl + +=head1 + +rename_locations.pl - renaming locations in a cxgn database + +=head1 SYNOPSIS + + rename_locations.pl -H [dbhost] -D [dbname] -i [infile] + +=head1 COMMAND-LINE OPTIONS + ARGUMENTS + -H host name (required) e.g. "localhost" + -D database name (required) e.g. "cxgn_cassava" + -i path to infile (required) + +=head1 DESCRIPTION + +This script loads locations data into Chado, by adding data to nd_geolocation table. Infile is Excel .xls and .xlsx format. +Header is in this order: 'old_location_name', 'new_location_name' + +=head1 AUTHOR + + Lukas Mueller (lam87@cornell.edu) + + Based on a script by Nicolas Morales (nm529@cornell.edu) + +=cut + +use strict; + +use Try::Tiny; +use Getopt::Std; +use Data::Dumper; +use Carp qw /croak/ ; +use Pod::Usage; +use Spreadsheet::ParseExcel; +use Spreadsheet::ParseXLSX; +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; + +our ($opt_H, $opt_D, $opt_i); + +getopts('H:D:i:'); + +if (!$opt_H || !$opt_D || !$opt_i) { + pod2usage(-verbose => 2, -message => "Must provide options -H (hostname), -D (database name), -i (input file) \n"); +} + +my $dbhost = $opt_H; +my $dbname = $opt_D; + +# Match a dot, extension .xls / .xlsx +my ($extension) = $opt_i =~ /(\.[^.]+)$/; +my $parser; + +if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); +} +else { + $parser = Spreadsheet::ParseExcel->new(); +} + +my $excel_obj = $parser->parse($opt_i); + +my $dbh = CXGN::DB::InsertDBH->new({ + dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 1, RaiseError => 1} +}); + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); +$dbh->do('SET search_path TO public,sgn'); + + +my $worksheet = ( $excel_obj->worksheets() )[0]; #support only one worksheet +my ( $row_min, $row_max ) = $worksheet->row_range(); +my ( $col_min, $col_max ) = $worksheet->col_range(); + +if ($col_max ne '1' || $worksheet->get_cell(0,0)->value() ne 'old_location_name' || $worksheet->get_cell(0,1)->value() ne 'new_location_name') { + pod2usage(-verbose => 2, -message => "Headers must be only in this order: old_location_name, new_location_name.\n"); +} + + +try { + $schema->txn_do( + sub { + for my $row ( 1 .. $row_max ) { + my $old_name = $worksheet->get_cell($row,0)->value(); + my $new_name = $worksheet->get_cell($row,1)->value(); + my $row = $schema->resultset('NaturalDiversity::NdGeolocation')->find({ description => $old_name }); + if ($row) { + $row->description($new_name); + $row->update(); + print STDERR "Updated $old_name to $new_name\n"; + } + else { + print STDERR "Location $old_name was not found. Skipping.\n"; + } + } + }); +} + +catch { + my $error = shift; + print STDERR "An error occurred ($error). Rolling back.\n"; +}; + +print STDERR "Script Complete.\n"; diff --git a/bin/rename_trials.pl b/bin/rename_trials.pl new file mode 100644 index 0000000000..81cdab8d4d --- /dev/null +++ b/bin/rename_trials.pl @@ -0,0 +1,152 @@ +#!/usr/bin/perl + +=head1 NAME + +rename_trials.pl - a script for renaming trials + +=head1 SYNOPSIS + +rename_trials.pl -H [dbhost] -D [dbname] -i [infile] + +=head2 Command-line options + +=over 5 + +=item -H + +host name (required) e.g. "localhost" + +=item -D + +database name (required) e.g. "cxgn_cassava" + +=item -i + +path to infile (required) + +=item -s + +stock type (default: accession) + +=item -n + +don't store old name as a synonym + +=item -t + +test mode, do not commit changes. + +=back + +=head1 DESCRIPTION + +This script renames trials in bulk using an xls and xlsx files as input with two columns: the first column is the old projectname as it is in the database, and in the second column is the new projectname. There is no header line. Both stock.name and stock.projectname fields will be changed to the new name. + +#The oldname will be stored as a synonym unless option -n is given. + +=head1 AUTHORS + +Guillaume Bauchet (gjb99@cornell.edu) + +Lukas Mueller (added -n option) + +Adapted from a cvterm renaming script by: + +Nicolas Morales (nm529@cornell.edu) + +Srikanth Kumar K (sk2783@cornell.edu) + +=cut + +use strict; + +use Getopt::Std; +use Data::Dumper; +use Carp qw /croak/ ; +use Pod::Usage; +use Spreadsheet::ParseExcel; +use Spreadsheet::ParseXLSX; +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; +use Try::Tiny; +use SGN::Model::Cvterm; + +our ($opt_H, $opt_D, $opt_i, $opt_s, $opt_t, $opt_n); + +getopts('H:D:i:s:tn'); + +if (!$opt_H || !$opt_D || !$opt_i) { + pod2usage(-verbose => 2, -message => "Must provide options -H (hostname), -D (database name), -i (input file)\n"); +} + +my $dbhost = $opt_H; +my $dbname = $opt_D; +my $stock_type = $opt_s || "accession"; + +# Match a dot, extension .xls / .xlsx +my ($extension) = $opt_i =~ /(\.[^.]+)$/; +my $parser; + +if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); +} +else { + $parser = Spreadsheet::ParseExcel->new(); +} + +my $excel_obj = $parser->parse($opt_i); + +my $dbh = CXGN::DB::InsertDBH->new({ + dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 0, RaiseError => 1} +}); + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); +$dbh->do('SET search_path TO public,sgn'); + +my $synonym_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); + + +my $worksheet = ( $excel_obj->worksheets() )[0]; #support only one worksheet +my ( $row_min, $row_max ) = $worksheet->row_range(); +my ( $col_min, $col_max ) = $worksheet->col_range(); + +my $coderef = sub { + for my $row ( 0 .. $row_max ) { + + my $db_projectname = $worksheet->get_cell($row,0)->value(); + my $new_projectname = $worksheet->get_cell($row,1)->value(); + + print STDERR "processing row $row: $db_projectname -> $new_projectname\n"; + + my $old_project = $schema->resultset('Project::Project')->find({ name => $db_projectname, projectname => $db_projectname }); + + if (!$old_project) { + print STDERR "Warning! Stock with projectname $db_projectname was not found in the database.\n"; + next(); + } + + my $new_project = $old_project->update({ name => $new_projectname }); + } +}; + +my $transaction_error; +try { + eval($coderef->()); +} catch { + $transaction_error = $_; +}; + +if ($opt_t) { + print STDERR "Not storing with test flag (-t). Rolling back.\n"; + $schema->txn_rollback(); +} +elsif ($transaction_error) { + print STDERR "Transaction error storing terms: $transaction_error. Rolling back.\n"; + $schema->txn_rollback(); +} else { + print STDERR "Everything looks good. Committing.\n"; + $schema->txn_commit(); + print STDERR "Script Complete.\n"; +} \ No newline at end of file diff --git a/bin/replace_plots.pl b/bin/replace_plots.pl index 32ed005b53..4ee3b95c5d 100644 --- a/bin/replace_plots.pl +++ b/bin/replace_plots.pl @@ -21,7 +21,7 @@ =head1 DESCRIPTION The infile provided has three columns: -1) the stock_id of the plot that needs to be changed. +1) the plot name that needs to be changed. 2) the stock uniquename as it is in the database that is currently associated 3) the new stock uniquename (needs to be in the database). 4) optional: a new plot name @@ -51,6 +51,7 @@ =head1 AUTHOR use Bio::Chado::Schema; use CXGN::DB::InsertDBH; use Try::Tiny; +use SGN::Model::Cvterm; our ($opt_H, $opt_D, $opt_i, $opt_t); @@ -94,13 +95,16 @@ =head1 AUTHOR for my $row ( 0 .. $row_max ) { my $plot_name = $worksheet->get_cell($row, 0)->value(); - my $plot_id = $worksheet->get_cell($row, 1)->value(); - my $db_uniquename = $worksheet->get_cell($row,2)->value(); - my $new_uniquename = $worksheet->get_cell($row,3)->value(); + my $plot_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'plot', 'stock_type')->cvterm_id(); + my $plot_id = $schema->resultset('Stock::Stock')->find({uniquename=>$plot_name, type_id=>$plot_cvterm_id})->stock_id(); + + # my $plot_id = $worksheet->get_cell($row, 1)->value(); + my $db_uniquename = $worksheet->get_cell($row,1)->value(); + my $new_uniquename = $worksheet->get_cell($row,2)->value(); my $new_plotname = ""; eval { - $new_plotname = $worksheet->get_cell($row, 4)->value(); + $new_plotname = $worksheet->get_cell($row, 3)->value(); }; if ($@) { diff --git a/bin/replace_stockprop_values.pl b/bin/replace_stockprop_values.pl new file mode 100644 index 0000000000..57a7f864c3 --- /dev/null +++ b/bin/replace_stockprop_values.pl @@ -0,0 +1,210 @@ +#!/usr/bin/perl + +=head1 NAME + +replace_stockprop_values.pl - a bulk script to replace phenotypic values in the database + +=head1 SYNOPSIS + +replace_stockprop_values.pl -H [dbhost] -D [dbname] -i [infile] <-t> + +=head1 COMMAND-LINE OPTIONS + +-H host name (required) e.g. "localhost" +-D database name (required) e.g. "cxgn_cassava" +-i path to infile (required) +-s stock type (default plot) +-t (optional) test - do not modify the database. + +=head1 DESCRIPTION + +This script replaces stockprops in bulk. + +The infile is an Excel file with three columns: + +1) the plot_name of the measurement to be replaced +2) in the header, " old" with the old stockprops in the column +3) the new value, in the header " new" with the new stockprops + in the rest of the column + +=head1 AUTHORS + + Lukas Mueller (lam87@cornell.edu) + + Adapted from a stock re-assigning script, which is based on other + scripts, originally by: + Guillaume Bauchet (gjb99@cornell.edu) + Nicolas Morales (nm529@cornell.edu) + +=cut + +use strict; + +use Try::Tiny; +use Getopt::Std; +use Data::Dumper; +use Carp qw /croak/ ; +use Pod::Usage; +use Spreadsheet::ParseExcel; +use Spreadsheet::ParseXLSX; +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; +use SGN::Model::Cvterm; + +our ($opt_H, $opt_D, $opt_i, $opt_t, $opt_s); + +getopts('H:D:i:ts:'); + +if (!$opt_H || !$opt_D || !$opt_i) { + pod2usage(-verbose => 2, -message => "Must provide options -H (hostname), -D (database name), -i (input file)\n"); +} + +my $dbhost = $opt_H; +my $dbname = $opt_D; +my $stock_type = $opt_s || 'plot'; + +# Match a dot, extension .xls / .xlsx +my ($extension) = $opt_i =~ /(\.[^.]+)$/; +my $parser; + +if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); +} +else { + $parser = Spreadsheet::ParseExcel->new(); +} + +my $excel_obj = $parser->parse($opt_i); + +my $dbh = CXGN::DB::InsertDBH->new({ + dbhost=>$dbhost, + dbname=>$dbname, + dbargs => {AutoCommit => 1, RaiseError => 1} +}); + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() } ); +$dbh->do('SET search_path TO public,sgn'); + +my $stock_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, $stock_type, "stock_type")->cvterm_id(); +print STDERR "Retrieved plot cvterm id of $stock_cvterm_id\n"; + +my $worksheet = ( $excel_obj->worksheets() )[0]; #support only one worksheet + +my $stockprop_name_old = $worksheet->get_cell(0, 1)->value(); +my $stockprop_name_new = $worksheet->get_cell(0, 2)->value(); + + +$stockprop_name_old =~ s/ old//g; + +$stockprop_name_new =~ s/ new//g; + +if ($stockprop_name_old ne $stockprop_name_new) { + die "The old and new stockprop names don't match. Please correct this and try again."; +} + +print STDERR "Working with stockprop called $stockprop_name_new ...\n"; +my $stockprop_cvterm = SGN::Model::Cvterm->get_cvterm_row($schema, $stockprop_name_new, "stock_property"); + +if (! $stockprop_cvterm) { + die "The cvterm $stockprop_name_new does not exist as a stockprop in this database. Please use another stock property"; +} + +my $stockprop_cvterm_id = $stockprop_cvterm->cvterm_id(); + +my ( $row_min, $row_max ) = $worksheet->row_range(); +my ( $col_min, $col_max ) = $worksheet->col_range(); + +my $coderef = sub { + + print STDERR "Parsing file ...\n"; + for my $row ( 1 .. $row_max ) { + + my $plot_name = $worksheet->get_cell($row, 0)->value(); + my $old_values = $worksheet->get_cell($row, 1)->value(); + my $new_values = $worksheet->get_cell($row,2)->value(); + + my $plot_row = $schema->resultset('Stock::Stock')->find({ uniquename => $plot_name, type_id => $stock_cvterm_id }); + + if (! $plot_row) { + print STDERR "Warning! Plot with uniquename $plot_name was not found in the database.\n"; + next(); + } + else { + print STDERR "FOUND PLOT $plot_name... OLD VALUES: $old_values. NEW VALUES: $new_values\n"; + } + + my @old_notes = split /\t/, $old_values; + my @new_notes = split /\t/, $new_values; + + print STDERR "OLD NOTES: ".Dumper(\@old_notes); + + for (my $i =0; $i < @old_notes; $i++) { + + + my $old_value = $old_notes[$i]; + + print STDERR "Working with note $old_value\n"; + + my $q = "select stockprop.stockprop_id, stockprop.value from stock join stockprop using(stock_id) where stock.uniquename=? and stockprop.type_id=? and value=?"; + + my $h = $dbh->prepare($q); + $h->execute($plot_name, $stockprop_cvterm_id, $old_value); + + my @rows = (); + + while (my ($phenotype_id, $db_value) = $h->fetchrow_array()) { + push @rows, [ $phenotype_id, $db_value ]; + } + + my $found_old_value = 0; + my $stockprop_id; + + foreach my $r (@rows) { + if ($r->[1] eq $old_value) { + $stockprop_id = $r->[0]; + $found_old_value =1; + } + } + + if ($found_old_value) { + print STDERR "FOUND OLD VALUE $old_value.\n"; + } + else { + print STDERR "DID NOT FIND OLD VALUE $old_value IN DATABASE.\n"; + } + + if ( (@rows > 1) && (!$found_old_value) ) { + print STDERR "MULTIPLE NOTES ARE ASSOCIATED WITH PLOT $plot_name AND TRAIT VARIABLE $stockprop_name_new\n"; + + } + + #update stock_relationship row with new object_id... + my $uq = "UPDATE stockprop set value=? where stockprop_id=?"; + + my $uh = $dbh->prepare($uq); + + if ($found_old_value) { + print STDERR "UPDATING WITH VALUE $new_notes[$i]\n"; + $uh->execute($new_notes[$i], $stockprop_id); + } + else { + print STDERR "NOT UPDATING, AS OLD VALUE NOT FOUND.\n"; + } + } + } +}; + + +my $transaction_error; +try { + if ($opt_t) { die "opt t : not saving!\n";} + $schema->txn_do($coderef); +} catch { + $transaction_error = $_; +}; + +if ($transaction_error) { + print STDERR "Transaction error storing terms: $transaction_error\n"; +} else { + print STDERR "Script Complete.\n"; +} diff --git a/bin/update_accession_species.pl b/bin/update_accession_species.pl new file mode 100644 index 0000000000..e254a20cd5 --- /dev/null +++ b/bin/update_accession_species.pl @@ -0,0 +1,151 @@ +#!/usr/bin/perl + +=head1 NAME + +update_stock_props.pl - updates stock props + +=head1 DESCRIPTION + +update_accession_species.pl -H [database host] -D [database name] -i update_file.xlsx + +Options: + + -H the database host + -D the database name + -i update_file.xlsx + +update_stock_prop_file.xlsx: a file with two columns: + accession_name + species_name + +=head1 AUTHOR + +Lukas Mueller new( { dbhost=>"$dbhost", + dbname=>"$dbname", + dbargs => {AutoCommit => 1, + RaiseError => 1, + } + } ); + +$dbh->do('set search_path to public'); + +my $schema= Bio::Chado::Schema->connect( sub { $dbh->get_actual_dbh() }); + +#my $formula_cvterm = $schema->resultset("Cv::Cvterm")->create_with({ +# name => "formula", +# cv => "cvterm_property", +#}); + +#my $type_id = $formula_cvterm->cvterm_id(); + + +# Match a dot, extension .xls / .xlsx +my ($extension) = $file =~ /(\.[^.]+)$/; +my $parser; + +if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); +} +else { + $parser = Spreadsheet::ParseExcel->new(); +} + +#try to open the excel file and report any errors +my $excel_obj = $parser->parse($file); + +if ( !$excel_obj ) { + die "Input file error: ".$parser->error()."\n"; +} + +my $worksheet = ( $excel_obj->worksheets() )[0]; #support only one worksheet +my ( $row_min, $row_max ) = $worksheet->row_range(); +my ( $col_min, $col_max ) = $worksheet->col_range(); + +if (($col_max - $col_min) < 1 || ($row_max - $row_min) < 1 ) { #must have header and at least one row of phenotypes + die "Input file error: spreadsheet is missing header\n"; +} + +# read header line +# +my ($accession_header, $species_header); + +if ($worksheet->get_cell(0,0)) { + $accession_header = $worksheet->get_cell(0,0)->value(); + if ($accession_header ne 'accession_name') { die "accession header not found"; } +} +if ($worksheet->get_cell(0,1)) { + $species_header = $worksheet->get_cell(0,1)->value(); + if ($species_header ne 'species_name') { die "species header not found" }; +} +#if ($worksheet->get_cell(0,2)) { +# $cvterm_new = $worksheet->get_cell(0,2)->value(); +# $cvterm_new =~ s/(.*)\_new/$1/g; +# if (! $cvterm_new) { die "cvterm needs to be cvterm with _new extension in old column"; } +#} + +#if ($cvterm_new ne $cvterm_old) { + # die "cvterm_new must be the same as cvterm_old without the extension, currently $cvterm_new vs $cvterm_old"; +#} + +#print STDERR "Working with stock property $cvterm_new\n"; +#my $cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, $cvterm_new, 'stock_property')->cvterm_id(); + +# read data lines +# +#my ($cvterm_new_value, $cvterm_old_value); +my ($accession_name, $species); +for (my $n=1; $n<=$row_max; $n++) { + + if ($worksheet->get_cell($n,0)) { + $accession_name = $worksheet->get_cell($n,0)->value(); + } + if ($worksheet->get_cell($n,1)) { + $species = $worksheet->get_cell($n,1)->value(); + } +# if ($worksheet->get_cell($n,2)) { +# $cvterm_new_value = $worksheet->get_cell($n,2)->value(); +# } + + my $accession_row = $schema->resultset("Stock::Stock")->find( { uniquename => $accession_name } ); + if (!$accession_row) { die "Accession $accession_name does not exist. Please fix and try again."; } + + my $organism_row = $schema->resultset("Organism::Organism")->find( { species => $species }); + + if (! $organism_row) { + die "The organism $species does not exit in the database"; + } + + my $current_organism_id = $accession_row->organism_id(); + + print STDERR "Accession: ".$accession_row->uniquename().". Current organism_id: $current_organism_id. New organism: ".$organism_row->organism_id()." ($species)\n"; + + $accession_row->organism_id($organism_row->organism_id()); + $accession_row->update(); +} + + +print STDERR "Done.\n"; diff --git a/bin/upload_multiple_trial_design.pl b/bin/upload_multiple_trial_design.pl new file mode 100644 index 0000000000..04a423572e --- /dev/null +++ b/bin/upload_multiple_trial_design.pl @@ -0,0 +1,297 @@ +#!/usr/bin/perl + +=head1 + +upload_multiple_trial_design.pl + +=head1 SYNOPSIS + +upload_multiple_trial_design.pl -H [dbhost] -D [dbname] -P [dbpass] -w [basepath] -U [dbuser] -i infile -un [username] -e [email address] + +=head1 COMMAND-LINE OPTIONS +ARGUMENTS + -H host name (required) Ex: "breedbase_db" + -D database name (required) Ex: "breedbase" + -U database username Ex: "postgres" + -P database userpass Ex: "postgres" + -w basepath (required) Ex: /home/production/cxgn/sgn + -i path to infile (required) + -un username of uploader (required) + -e email address of the user +if loading trial data from metadata file, phenotypes + layout from infile + +=head2 DESCRIPTION + +perl bin/upload_multiple_trial_design.pl -H breedbase_db -D breedbase -U postgres -P postgres -w /home/cxgn/sgn/ -un janedoe -i ~/Desktop/test_multi.xlsx -e 'sk2783@cornell.edu' -iw + +This script will parse and validate the input file. If there are any warnings or errors during validation it will send a error message to the provided email. It will print any errors and warnings to the console. +If there are no errors or warnings (or warnings are ignored) during validation it will then store the data. +The input file should be any file supported by the CXGN::File::Parse class. + +=head1 AUTHOR + +Srikanth (sk2783@cornell.edu) + +=cut + +use strict; +use Getopt::Long; +use Bio::Chado::Schema; +use CXGN::DB::InsertDBH; +use Try::Tiny; +use DateTime; +use Pod::Usage; +use CXGN::Trial; # add project metadata +use CXGN::Trial::ParseUpload; +use CXGN::Trial::TrialCreate; +use CXGN::Contact; +use CXGN::TrialStatus; + +my ( $help, $dbhost, $dbname, $basepath, $dbuser, $dbpass, $infile, $username, $email_address, $ignore_warnings); +GetOptions( + 'dbhost|H=s' => \$dbhost, + 'dbname|D=s' => \$dbname, + 'dbuser|U=s' => \$dbuser, + 'dbpass|P=s' => \$dbpass, + 'basepath|w=s' => \$basepath, + 'i=s' => \$infile, + 'user|un=s' => \$username, + 'email|e=s' => \$email_address, + 'ignore_warnings|iw!' => \$ignore_warnings, + 'help' => \$help, +); +pod2usage(1) if $help; +if (!$infile || !$username || !$basepath || !$dbname || !$dbhost ) { + pod2usage({ -msg => 'Error. Missing options!', -verbose => 1, -exitval => 1 }); +} + +# Lists of encountered errors and warnings +my @errors; +my @warnings; + +# Connect to databases +my $dbh; +if ($dbpass && $dbuser) { + $dbh = DBI->connect( + "dbi:Pg:database=$dbname;host=$dbhost", + $dbuser, + $dbpass, + {AutoCommit => 1, RaiseError => 1} + ); +} +else { + $dbh = CXGN::DB::InsertDBH->new({ + dbhost => $dbhost, + dbname => $dbname, + dbargs => {AutoCommit => 1, RaiseError => 1} + }); +} +my $chado_schema = Bio::Chado::Schema->connect(sub { $dbh }, { on_connect_do => ['SET search_path TO public, sgn, metadata, phenome;'] }); +print STDOUT "Database connection ok!\n"; + +my $parsed_data; +my $validation_coderef = sub { + # Parse uploaded file with appropriate plugin + my $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $infile); + $parser->load_plugin('MultipleTrialDesignGeneric'); + $parsed_data = $parser->parse(); + + # Parser has errors, print error messages and quit + if ($parser->has_parse_errors()) { + my $errors = $parser->get_parse_errors(); + foreach (@{$errors->{'error_messages'}}) { + push @errors, $_; + } + finish(); + } + + # Parser has warnings, print warning messages and quit unless we're ignoring warnings + if ($parser->has_parse_warnings()) { + unless ($ignore_warnings) { + my $warnings = $parser->get_parse_warnings(); + foreach (@{$warnings->{'warning_messages'}}) { + push @warnings, $_; + } + finish(); + } + } +}; + +try { + $chado_schema->txn_do($validation_coderef); +} catch { + push @errors, $_; +}; + +# Check for parsed data +finish("There is no parsed data from the input file!") if !$parsed_data; + +# Get User ID +my $sp_person_id = CXGN::People::Person->get_person_by_username($dbh, $username); +finish("User not found in database for username $username!") if !$sp_person_id; + +# Create and Save Trials +my %all_designs = %{$parsed_data}; +my %saved_trials; +my $coderef = sub { + for my $trial_name ( keys %all_designs ) { + my $trial_design = $all_designs{$trial_name}; + my %trial_info_hash = ( + chado_schema => $chado_schema, + dbh => $dbh, + owner_id => $sp_person_id, + trial_year => $trial_design->{'year'}, + trial_description => $trial_design->{'description'}, + trial_location => $trial_design->{'location'}, + trial_name => $trial_name, + design_type => $trial_design->{'design_type'}, + trial_stock_type => $trial_design->{'trial_stock_type'}, + design => $trial_design->{'design_details'}, + program => $trial_design->{'breeding_program'}, + upload_trial_file => $infile, + operator => $username, + owner_id => $sp_person_id + ); + my $entry_numbers = $trial_design->{'entry_numbers'}; + + if ($trial_design->{'trial_type'}){ + $trial_info_hash{trial_type} = $trial_design->{'trial_type'}; + } + if ($trial_design->{'plot_width'}){ + $trial_info_hash{plot_width} = $trial_design->{'plot_width'}; + } + if ($trial_design->{'plot_length'}){ + $trial_info_hash{plot_length} = $trial_design->{'plot_length'}; + } + if ($trial_design->{'field_size'}){ + $trial_info_hash{field_size} = $trial_design->{'field_size'}; + } + if ($trial_design->{'planting_date'}){ + $trial_info_hash{planting_date} = $trial_design->{'planting_date'}; + } + if ($trial_design->{'harvest_date'}){ + $trial_info_hash{harvest_date} = $trial_design->{'harvest_date'}; + } + if ($trial_design->{'transplanting_date'}){ + $trial_info_hash{transplanting_date} = $trial_design->{'transplanting_date'}; + } + my $trial_create = CXGN::Trial::TrialCreate->new(\%trial_info_hash); + my $current_save = $trial_create->save_trial(); + + if ($current_save->{error}){ + $chado_schema->txn_rollback(); + finish($current_save->{'error'}); + } elsif ($current_save->{'trial_id'}) { + my $trial_id = $current_save->{'trial_id'}; + my $time = DateTime->now(); + my $timestamp = $time->ymd(); + my $calendar_funcs = CXGN::Calendar->new({}); + my $formatted_date = $calendar_funcs->check_value_format($timestamp); + my $upload_date = $calendar_funcs->display_start_date($formatted_date); + $saved_trials{$trial_id} = $trial_name; + + my %trial_activity; + $trial_activity{'Trial Uploaded'}{'user_id'} = $sp_person_id; + $trial_activity{'Trial Uploaded'}{'activity_date'} = $upload_date; + + my $trial_activity_obj = CXGN::TrialStatus->new({ bcs_schema => $chado_schema }); + $trial_activity_obj->trial_activities(\%trial_activity); + $trial_activity_obj->parent_id($trial_id); + my $activity_prop_id = $trial_activity_obj->store(); + } + + # save entry numbers, if provided + if ( $entry_numbers && scalar(keys %$entry_numbers) > 0 && $current_save->{'trial_id'} ) { + my %entry_numbers_prop; + my @stock_names = keys %$entry_numbers; + + # Convert stock names from parsed trial template to stock ids for data storage + my $stocks = $chado_schema->resultset('Stock::Stock')->search({ uniquename=>{-in=>\@stock_names} }); + while (my $s = $stocks->next()) { + $entry_numbers_prop{$s->stock_id} = $entry_numbers->{$s->uniquename}; + } + + # Lookup synonyms of accession names + my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($chado_schema, 'stock_synonym', 'stock_property')->cvterm_id(); + my $acc_synonym_rs = $chado_schema->resultset("Stock::Stock")->search({ + 'me.is_obsolete' => { '!=' => 't' }, + 'stockprops.value' => { -in => \@stock_names}, + 'stockprops.type_id' => $synonym_cvterm_id + },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); + while (my $r=$acc_synonym_rs->next) { + if ( exists($entry_numbers->{$r->get_column('synonym')}) ) { + $entry_numbers_prop{$r->stock_id} = $entry_numbers->{$r->get_column('synonym')}; + } + } + + # store entry numbers + my $trial = CXGN::Trial->new({ bcs_schema => $chado_schema, trial_id => $current_save->{'trial_id'} }); + $trial->set_entry_numbers(\%entry_numbers_prop); + } + } + +}; + +try { + $chado_schema->txn_do($coderef); +} catch { + push @errors, $_; +}; + +finish(); + +sub finish { + my $error = shift; + push @errors, $error if $error; + + # Print errors and warnings to STDERR + foreach (@errors) { + print STDERR "ERROR: $_\n"; + } + foreach (@warnings) { + print STDERR "WARNING: $_\n"; + } + + # Send email message, if requested + # Exit the script: 0 = success, 1 = errors, 2 = warnings + if ( scalar(@errors) > 0 ) { + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload failed"; + my $email_body = "Dear $username,\n\nThere were one or more errors in uploading your trials:\n\n"; + foreach my $error (@errors) { + $error =~ s/<[^>]*>//g; + $email_body .= "$error\n"; + } + $email_body .= "\nYou will need to fix the errors and upload the corrected file. Thank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + exit(1); + } + if ( scalar(@warnings) > 0 ) { + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload failed"; + my $email_body = "Dear $username,\n\nThere were one or more warnings in uploading your trials and the option to ignore warnings was not enabled. The warnings include:\n\n"; + foreach my $warning (@warnings) { + $warning =~ s/<[^>]*>//g; + $email_body .= "$warning\n"; + } + $email_body .= "\nYou will need to either fix the warnings and upload the corrected file or upload the same file with the option to ignore warnings enabled. Thank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + exit(2); + } + else { + my $bs = CXGN::BreederSearch->new({ dbh=>$dbh, dbname=>$dbname }); + my $refresh = $bs->refresh_matviews($dbhost, $dbname, $dbuser, $dbpass, 'all_but_genoview', 'concurrent', $basepath); + + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload successful"; + my $email_body = "Dear $username,\n\nCongratulations, all the multiple trial designs have been successfully uploaded into the database\n\nThank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + + exit(0); + } +} + +1; \ No newline at end of file diff --git a/cgi-bin/chado/organism_query.pl b/cgi-bin/chado/organism_query.pl index 36c30e7a39..79309939d2 100644 --- a/cgi-bin/chado/organism_query.pl +++ b/cgi-bin/chado/organism_query.pl @@ -13,7 +13,8 @@ my $json = JSON->new(); -my $schema = $c->dbic_schema('Bio::Chado::Schema', 'sgn_chado'); +my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; +my $schema = $c->dbic_schema('Bio::Chado::Schema', 'sgn_chado', $sp_person_id); my $org_rs = $schema->resultset("Organism::Organism")->search({species=>{'ilike'=>'%'.$species.'%'}}); diff --git a/cgi-bin/genomes/Solanum_lycopersicum/publications.pl b/cgi-bin/genomes/Solanum_lycopersicum/publications.pl index 94c87cf5f3..3e314c4d05 100644 --- a/cgi-bin/genomes/Solanum_lycopersicum/publications.pl +++ b/cgi-bin/genomes/Solanum_lycopersicum/publications.pl @@ -2,5 +2,6 @@ use strict; use warnings; -my $schema = $c->dbic_schema('Bio::Chado::Schema'); +my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; +my $schema = $c->dbic_schema('Bio::Chado::Schema', undef, $sp_person_id); $c->forward_to_mason_view('/genomes/Solanum_lycopersicum/publications.mas', schema => $schema); diff --git a/cgi-bin/jsforms/pub_ajax_form.pl b/cgi-bin/jsforms/pub_ajax_form.pl index b9495d6ab4..4191293295 100644 --- a/cgi-bin/jsforms/pub_ajax_form.pl +++ b/cgi-bin/jsforms/pub_ajax_form.pl @@ -35,7 +35,8 @@ sub define_object { my $user_type = $self->get_user()->get_user_type(); my %json_hash= $self->get_json_hash(); - my $schema = $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado' ); + my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; + my $schema = $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado', $sp_person_id ); my $dbh = $schema->storage->dbh; $self->set_object_id($pub_id); $self->set_object_name('publication'); #this is useful for email messages diff --git a/cgi-bin/phenome/assign_owner.pl b/cgi-bin/phenome/assign_owner.pl index 8c8202505d..c09cb3e2f6 100755 --- a/cgi-bin/phenome/assign_owner.pl +++ b/cgi-bin/phenome/assign_owner.pl @@ -84,7 +84,7 @@ } elsif ( $object_type eq 'stock' ) { my $stock = - $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado' ) + $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado', $sp_person_id ) ->resultset("Stock::Stock") ->find( { stock_id => $object_id } ); $stock->create_stockprops( diff --git a/cgi-bin/phenome/associate_allele.pl b/cgi-bin/phenome/associate_allele.pl index 8590c41b3a..12a131d067 100755 --- a/cgi-bin/phenome/associate_allele.pl +++ b/cgi-bin/phenome/associate_allele.pl @@ -13,7 +13,8 @@ $doc->send_http_header(); my $dbh = $c->dbc->dbh; -my $schema = $c->dbic_schema('Bio::Chado::Schema' , 'sgn_chado'); +my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; +my $schema = $c->dbic_schema('Bio::Chado::Schema' , 'sgn_chado', $sp_person_id); my($login_person_id,$login_user_type)=CXGN::Login->new($dbh)->verify_session(); if ($login_user_type eq 'curator' || $login_user_type eq 'submitter' || $login_user_type eq 'sequencer') { diff --git a/cgi-bin/phenome/locus_browser.pl b/cgi-bin/phenome/locus_browser.pl index aec0cc47f7..5d60c793af 100755 --- a/cgi-bin/phenome/locus_browser.pl +++ b/cgi-bin/phenome/locus_browser.pl @@ -25,7 +25,8 @@ my $dbh = CXGN::DB::Connection->new(); -my $schema= $c->dbic_schema('CXGN::Phenome::Schema'); +my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; +my $schema= $c->dbic_schema('CXGN::Phenome::Schema', undef, $sp_person_id); my ($login_person_id, $login_user_type)=CXGN::Login->new($dbh)->verify_session(); diff --git a/cgi-bin/phenome/stock/stock_ajax_form.pl b/cgi-bin/phenome/stock/stock_ajax_form.pl index 980df5ef1f..1fb5601325 100644 --- a/cgi-bin/phenome/stock/stock_ajax_form.pl +++ b/cgi-bin/phenome/stock/stock_ajax_form.pl @@ -33,8 +33,11 @@ sub define_object { my $stock_id = $args{stock_id} || $args{object_id}; my $user_type = $self->get_user()->get_user_type(); my %json_hash= $self->get_json_hash(); + my $sp_person_id = $self->get_user()->get_sp_person_id(); - my $schema = $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado' ); + print STDERR "this is sp_person_id before dbic_schema: ".$sp_person_id."\n"; + + my $schema = $c->dbic_schema( 'Bio::Chado::Schema', 'sgn_chado', $sp_person_id); $self->set_object_id($stock_id); $self->set_object_name('Stock'); #this is useful for email messages diff --git a/cgi-bin/search/unigene.pl b/cgi-bin/search/unigene.pl index 166bd52bf5..0444b81987 100755 --- a/cgi-bin/search/unigene.pl +++ b/cgi-bin/search/unigene.pl @@ -41,8 +41,9 @@ =head1 AUTHORS my %args = $q->Vars; -my $schema = $c->dbic_schema( 'CXGN::GEM::Schema', 'sgn_chado' ); -my $sgn_schema = $c->dbic_schema( 'SGN::Schema' ); +my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; +my $schema = $c->dbic_schema( 'CXGN::GEM::Schema', 'sgn_chado', $sp_person_id ); +my $sgn_schema = $c->dbic_schema( 'SGN::Schema', undef, $sp_person_id ); my $dbh = CXGN::DB::Connection->new(); diff --git a/cgi-bin/tools/sixframe_translate.pl b/cgi-bin/tools/sixframe_translate.pl index ec7893c249..71a4bfda34 100755 --- a/cgi-bin/tools/sixframe_translate.pl +++ b/cgi-bin/tools/sixframe_translate.pl @@ -23,10 +23,12 @@ sub get_legacy_est { my ($cgi) = @_; + my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; + my $est_id = $cgi->param('est_id') or return; - my $est = $c->dbic_schema('SGN::Schema') + my $est = $c->dbic_schema('SGN::Schema', undef, $sp_person_id) ->resultset('Est') ->find( $est_id ) or return; @@ -39,10 +41,12 @@ sub get_legacy_est { sub get_legacy_unigene { my ($cgi) = @_; + my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef; + my $unigene_id = $cgi->param('unigene_id') or return; - my $unigene = $c->dbic_schema('SGN::Schema') + my $unigene = $c->dbic_schema('SGN::Schema', undef, $sp_person_id) ->resultset('Unigene') ->find( $unigene_id ) or return; diff --git a/cgi-bin/tools/tree_browser/index.pl b/cgi-bin/tools/tree_browser/index.pl index f31bfa8efb..52ea483df4 100755 --- a/cgi-bin/tools/tree_browser/index.pl +++ b/cgi-bin/tools/tree_browser/index.pl @@ -686,9 +686,10 @@ =head1 AUTHOR splice @ops, $i, 1; } } - - my $tooltip = $n->get_tooltip() - or "Node " . $n->get_node_key() . ": " . $n->get_label()->get_name(); + my $tooltip = $n->get_tooltip(); + if (!$tooltip) { + $tooltip = "Node " . $n->get_node_key() . ": " . $n->get_label()->get_name(); + } # my $tooltip = "Node " . $n->get_node_key(); my $model = $n->get_attribute("model"); diff --git a/db/00183/AddStockPropValueJsonb.pm b/db/00183/AddStockPropValueJsonb.pm new file mode 100644 index 0000000000..e37d300969 --- /dev/null +++ b/db/00183/AddStockPropValueJsonb.pm @@ -0,0 +1,72 @@ +#!/usr/bin/env perl + + +=head1 NAME + + AddStockPropValueJsonb; + +=head1 SYNOPSIS + +mx-run AddStockPropValueJsonb [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This is a test dummy patch. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddStockPropValueJsonb; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This dbpatch adds a value_jsonb field to the stockprop table to enable use of the jsonb format for faster access + +has '+prereq' => ( + default => sub { + [], + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds transformation_project project_type, transformation_experiment experiment_type, folder_for_transformations project_property, transformation stock_type, plant_material_of, vector_construct_of, transformant_of stock_relationship cvterms. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddTransformationRelatedCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Description of this patch goes here + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'project_type' => [ + 'transformation_project' + ], + 'project_property' => [ + 'folder_for_transformations' + ], + 'experiment_type' => [ + 'transformation_experiment' + ], + 'stock_type' => [ + 'transformation' + ], + 'stock_relationship' => [ + 'plant_material_of', + 'vector_construct_of', + 'transformant_of' + ], + 'stock_property' => [ + 'transformation_notes', + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; + +} + + +#### +1; # +#### diff --git a/db/00185/AddTrackingActivityRelatedCvterms.pm b/db/00185/AddTrackingActivityRelatedCvterms.pm new file mode 100644 index 0000000000..fbfde58d8d --- /dev/null +++ b/db/00185/AddTrackingActivityRelatedCvterms.pm @@ -0,0 +1,109 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddTrackingActivityRelatedCvterms.pm + +=head1 SYNOPSIS + +mx-run ThisPackageName [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds activity_record project_type cvterm, tracking_activity experiment_type cvterm, tracking_identifier stock_type cvterm, material_of stock_relationship cvterm, tracking_tissue_culture_json stock_property cvterm and tracking_identifiers list_type cvterm. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddTrackingActivityRelatedCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Description of this patch goes here + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'project_type' => [ + 'activity_record' + ], + 'project_property' => [ + 'activity_type', + 'project_vendor', + 'folder_for_tracking_activities' + ], + 'experiment_type' => [ + 'tracking_activity' + ], + 'stock_type' => [ + 'tracking_identifier' + ], + 'stock_relationship' => [ + 'material_of' + ], + 'stock_property' => [ + 'tracking_tissue_culture_json', + 'terminated_metadata', + ], + 'list_types' => [ + 'tracking_identifiers' + ], + 'sp_order_property' => [ + 'order_tracking_identifiers' + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; + +} + + +#### +1; # +#### diff --git a/db/00186/AddAuditTables.pm b/db/00186/AddAuditTables.pm new file mode 100644 index 0000000000..5aeab89175 --- /dev/null +++ b/db/00186/AddAuditTables.pm @@ -0,0 +1,3947 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddAuditTables.pm + +=head1 SYNOPSIS + +mx-run AddAuditTables.pm [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch adds audit tables +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Adrian Powell + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddAuditTables; + +use Moose; +use Bio::Chado::Schema; +use SGN::Model::Cvterm; + +extends 'CXGN::Metadata::Dbpatch'; + +has '+description' => ( default => <<'' ); +This patch adds audit tables + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch adds vendor role on sgn_people.sp_roles + +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Chris Simoes < ccs263@cornell.edu > + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddBreederRole; + +use Moose; + +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds vendor role on sgn_people.sp_roles + +has '+prereq' => ( + default => sub { + [], + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds pcr_marker_details, pcr_marker_genotyping and pcr_marker_protocol cvterms +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Isaak Y Tecle + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddPrcompCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'prcomp' protocol_type cvterm. + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'protocol_type' => [ + 'prcomp', + ] + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00187/AddTransformationAndTrackingCvterms.pm b/db/00187/AddTransformationAndTrackingCvterms.pm new file mode 100644 index 0000000000..21f856a3da --- /dev/null +++ b/db/00187/AddTransformationAndTrackingCvterms.pm @@ -0,0 +1,94 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddTransformationAndTrackingCvterms.pm + +=head1 SYNOPSIS + +mx-run ThisPackageName [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds autogenerated_name_metadata, autogenerated_name_format, default_plant_material project_property cvterms, progress_of project_relationship cvterm, tracking_transformation_json, completed_metadata stock_property cvterms. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddTransformationAndTrackingCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Description of this patch goes here + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'project_property' => [ + 'autogenerated_name_metadata', + 'autogenerated_name_format', + 'default_plant_material', + ], + 'project_relationship' => [ + 'progress_of', + ], + 'stock_property' => [ + 'tracking_transformation_json', + 'completed_metadata', + ] + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; + +} + + +#### +1; # +#### diff --git a/db/00187/AddValidatePhenotypeCvterm.pm b/db/00187/AddValidatePhenotypeCvterm.pm new file mode 100644 index 0000000000..70d61a33cd --- /dev/null +++ b/db/00187/AddValidatePhenotypeCvterm.pm @@ -0,0 +1,86 @@ +#!/usr/bin/env perl + + +=head1 NAME + + AddValidatePhenotypeCvterm + +=head1 SYNOPSIS + +mx-run AddValidadePhenotypeCvterm [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds phenotype_property cvterm for storing a suppressed plot phenotype. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddValidatePhenotypeCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds phenotype_property cvterm for storing a suppressed plot phenotype. + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'project_property' => [ + 'validated_phenotype', + ] + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + +print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00188/AddMemberTypeCvterm.pm b/db/00188/AddMemberTypeCvterm.pm new file mode 100644 index 0000000000..1739fc78bf --- /dev/null +++ b/db/00188/AddMemberTypeCvterm.pm @@ -0,0 +1,73 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddMemberTypeCvterm + +=head1 SYNOPSIS + +mx-run AddMemberTypeCvterm [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds member_type stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddMemberTypeCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'member_type' stock_property cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'member_type', + cv => 'stock_property' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00188/AddSelectedDisplayImageCvterm.pm b/db/00188/AddSelectedDisplayImageCvterm.pm new file mode 100644 index 0000000000..acbb299cf0 --- /dev/null +++ b/db/00188/AddSelectedDisplayImageCvterm.pm @@ -0,0 +1,84 @@ +#!/usr/bin/env perl + + +=head1 NAME + + AddSelectedDisplayImageCvterm.pm + +=head1 SYNOPSIS + +mx-run ThisPackageName [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This adds selected_display_image stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Benjamin Maza + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddSelectedDisplayImageCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the selected_display_image stock_property cvterm. + +has '+prereq' => ( + default => sub { + [], + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'stock_property' => [ + 'selected_display_image', + ] + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00189/UpdateMaterializedPhenoviewSubplots.pm b/db/00189/UpdateMaterializedPhenoviewSubplots.pm new file mode 100644 index 0000000000..82bbece358 --- /dev/null +++ b/db/00189/UpdateMaterializedPhenoviewSubplots.pm @@ -0,0 +1,1175 @@ +#!/usr/bin/env perl + + +=head1 NAME + +UpdateMaterializedPhenoviewSubplots.pm + +=head1 SYNOPSIS + +mx-run UpdateMaterializedPhenoviewSubplots [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding subplots to the view + +=head1 AUTHOR + +Bryan Ellerbrock +David Waring + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package UpdateMaterializedPhenoviewSubplots; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding subplots to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds the necessary cvterms that are used for brapi germplasm, programs, and studies. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Isaak Y Tecle + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddAnalysisResultStockType; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Description of this patch goes here + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'stock_type' => [ + 'analysis_result', + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + +print "You're done!\n"; +} + + +#### +1; # +#### \ No newline at end of file diff --git a/db/00190/AddOrganismsToMaterializedview.pm b/db/00190/AddOrganismsToMaterializedview.pm new file mode 100644 index 0000000000..7f50c92307 --- /dev/null +++ b/db/00190/AddOrganismsToMaterializedview.pm @@ -0,0 +1,1500 @@ +#!/usr/bin/env perl + + +=head1 NAME + +AddOrganismsToMaterializedview.pm + + +=head1 SYNOPSIS + +mx-run AddOrganismsToMaterializedview [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding tissue samples to the view + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + +package AddOrganismsToMaterializedview; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding tissue sample to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding tissue samples to the view + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + +package UpdateMaterializedPhenoviewTissueSample; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding tissue sample to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds material_type stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddMaterialTypeCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'material_type' stock_property cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'material_type', + cv => 'stock_property' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00191/CreateFuzzyExtensionAndIndexes.pm b/db/00191/CreateFuzzyExtensionAndIndexes.pm new file mode 100644 index 0000000000..2a63574ae8 --- /dev/null +++ b/db/00191/CreateFuzzyExtensionAndIndexes.pm @@ -0,0 +1,75 @@ +#!/usr/bin/env perl + + +=head1 NAME + + CreateFuzzyExtension + +=head1 SYNOPSIS + +mx-run CreateFuzzyExtensionAndIndexes [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch creates the extension pg_trgm for use of the similarity function for fuzzy search, as well as GIN index and btree index on LOWER(uniquename) on the stock table. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Ben Maza + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package CreateFuzzyExtensionAndIndexes; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch creates the extension pg_trgm for use of the similarity function for fuzzy search, as well as GIN index and btree index on LOWER(uniquename) on the stock table. + +has '+prereq' => ( + default => sub { + [], + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This DB patch adds an sp_job table to sgn_people. This table tracks background job submissions. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Ryan Preble + +=head1 COPYRIGHT & LICENSE + +Copyright 2025 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddJobsTable; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; + +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Adds sp_job table to sgn_people for submitted job tracking + +has '+prereq' => ( + default => sub { + [], + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $terms = { + 'job_type' => [ + 'download', + 'upload', + 'report', + 'search', + 'cluster_analysis', + 'training_model', + 'selection_prediction', + 'multiple_models', + 'training_dataset', + 'kinship_analysis', + 'heritability_analysis', + 'solGWAS_analysis', + 'spatial_analysis', + 'pca_analysis', + 'stability_analysis', + 'mixed_model_analysis', + 'nirs_analysis', + 'tool_compatibility', + 'genomic_prediction', + 'sequence_analysis', + ] + }; + + foreach my $cv (keys %$terms){ + foreach my $term (@{$terms->{$cv}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $term, + cv => $cv, + }); + } + } + + #my $dbuser = `cat /home/production/volume/cxgn/sgn/sgn_local.conf | grep dbuser | sed -r 's/\w+\s//'`; + my $dbuser = 'web_usr'; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds metadata_ontology cvterm for the post composing tool +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddMetadataOnto; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'metadata_ontology' composable_cvtypes cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'metadata_ontology', + cv => 'composable_cvtypes' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00192/AddObsoletedStocksListTypeCvterm.pm b/db/00192/AddObsoletedStocksListTypeCvterm.pm new file mode 100644 index 0000000000..c239b51076 --- /dev/null +++ b/db/00192/AddObsoletedStocksListTypeCvterm.pm @@ -0,0 +1,73 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddObsoletedStocksListTypeCvterm + +=head1 SYNOPSIS + +mx-run AddObsoletedStocksListTypeCvterm [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds obsoleted_stocks list type cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddObsoletedStocksListTypeCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'obsoleted_stocks' list_types cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'obsoleted_stocks', + cv => 'list_types' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00193/AddBiochemOnto.pm b/db/00193/AddBiochemOnto.pm new file mode 100644 index 0000000000..7d39156f89 --- /dev/null +++ b/db/00193/AddBiochemOnto.pm @@ -0,0 +1,74 @@ +#!/usr/bin/env perl + + +=head1 NAME + +AddBiochemOnto + +=head1 SYNOPSIS + +mx-run AddBiochemOnto [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds biochem_ontology cvterm for the post composing tool +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddBiochemOnto; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'biochem_ontology' composable_cvtypes cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'biochem_ontology', + cv => 'composable_cvtypes' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00193/AddControlRelatedCvterms.pm b/db/00193/AddControlRelatedCvterms.pm new file mode 100644 index 0000000000..2a4e7a6e9b --- /dev/null +++ b/db/00193/AddControlRelatedCvterms.pm @@ -0,0 +1,86 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddControlRelatedCvterms + +=head1 SYNOPSIS + +mx-run AddControlRelatedCvterms [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds 'control_of' stock_relationship cvterm and is_a_transformation_control stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddControlRelatedCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'control_of' stock_relationship cvterm and is_a_transformation_control stock_property cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'stock_relationship' => [ + 'control_of', + ], + 'stock_property' => [ + 'is_a_transformation_control', + ] + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00193/AddSpATSSpatialModelType.pm b/db/00193/AddSpATSSpatialModelType.pm new file mode 100644 index 0000000000..59b27edcc3 --- /dev/null +++ b/db/00193/AddSpATSSpatialModelType.pm @@ -0,0 +1,81 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddSpATSSpatialModelType + +=head1 SYNOPSIS + +mx-run AddSpATSSpatialModelType [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds member_type stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Ryan Preble + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddSpATSSpatialModelType; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'spatial_model_SpATS' protocol_type cvterm as well as a corresponding cvterm for the SGNStatistics_ontology + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'spatial_model_SpATS', + cv => 'protocol_type' + }); + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'Adjusted Means from Spatial Correction using SpATS R', + cv => 'SGNStatistics_ontology' + }); + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'spatially_corrected_trait_adjustments_json', + cv => 'project_property' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00193/AddTraitPropRepeatType.pm b/db/00193/AddTraitPropRepeatType.pm new file mode 100644 index 0000000000..855583ac32 --- /dev/null +++ b/db/00193/AddTraitPropRepeatType.pm @@ -0,0 +1,102 @@ +#!/usr/bin/env perl + + +=head1 NAME + + AddTraitPropRepeatType + +=head1 SYNOPSIS + +mx-run AddTraitPropRepeatType [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This dbpatch adds the trait_repeat_type property to the trait_property cv. + + +=head1 AUTHOR + +Lukas Mueller + +=head1 COPYRIGHT & LICENSE + +Copyright 2024 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddTraitPropRepeatType; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; + +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +Description of this patch goes here + +has '+prereq' => ( + default => sub { [] + }, + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + my %cvterms = ( + 'trait_property' => [ 'trait_repeat_type' ], + ); + + + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + my $coderef = sub { + + + foreach my $cv_name ( keys %cvterms ) { + print "\nKEY = $cv_name \n\n"; + my @cvterm_names = @{$cvterms{ $cv_name } } ; + + foreach my $cvterm_name ( @cvterm_names ) { + print "cvterm= $cvterm_name \n"; + my $new_cvterm = $schema->resultset("Cv::Cvterm")->create_with( + { + name => $cvterm_name, + cv => $cv_name, + }); + } + } + }; + + try { + $schema->txn_do($coderef); + + } catch { + die "Load failed! " . $_ . "\n" ; + }; + + + + +print "You successfully added the new property 'trait_repeat_type'!\n"; +} + + +#### +1; # +#### diff --git a/db/00193/UpdateListTypeLocusCvterms.pm b/db/00193/UpdateListTypeLocusCvterms.pm new file mode 100644 index 0000000000..1896e97319 --- /dev/null +++ b/db/00193/UpdateListTypeLocusCvterms.pm @@ -0,0 +1,85 @@ +#!/usr/bin/env perl + +=head1 NAME + + UpdateListTypeLocusCvterms.pm + +=head1 SYNOPSIS + +mx-run UpdateListTypeLocusCvterms [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch adds locus name list type. + +This subclass uses L. The parent class uses L + +=head1 AUTHOR + + Clay Birkett + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package UpdateListTypeLocusCvterms; + +use Moose; +use Bio::Chado::Schema; +use SGN::Model::Cvterm; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + + +has '+description' => ( default => <<'' ); +This patch adds locus and locus_alleles list type + +has '+prereq' => ( + default => sub { + [], + }, + + ); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $terms = { + 'list_types' => [ + 'locus', + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00194/AddVariableOfRel.pm b/db/00194/AddVariableOfRel.pm new file mode 100644 index 0000000000..e2274ef96c --- /dev/null +++ b/db/00194/AddVariableOfRel.pm @@ -0,0 +1,87 @@ +#!/usr/bin/env perl + + +=head1 NAME + +AddVariableOfRel + +=head1 SYNOPSIS + +mx-run AddVariableOfRel [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds VARIABLE_OF, method_of, scale_of cvterms using the relationship ontology cv +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddVariableOfRel; + +use Moose; +use Bio::Chado::Schema; +use SGN::Model::Cvterm; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the following cvterms to the relationship ontology: VARIABLE_OF, method_of, scale_of + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $coderef = sub { + print STDERR "INSERTING CVTERMS...\n"; + my @terms = ("VARIABLE_OF", "method_of", "scale_of"); + foreach my $term (@terms) { + my $new_cvterm = $schema->resultset("Cv::Cvterm")->create_with({ + name => $term, + is_relationshiptype => 1, + cv => 'relationship', + dbxref => $term, + db => 'OBO_REL' + }); + } + return 1; + }; + try { + $schema->txn_do($coderef); + } catch { + die "Load failed! " . $_ . "\n" ; + }; + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00195/UpdateTraitsView.pm b/db/00195/UpdateTraitsView.pm new file mode 100644 index 0000000000..1e332d24de --- /dev/null +++ b/db/00195/UpdateTraitsView.pm @@ -0,0 +1,90 @@ +#!/usr/bin/env perl + + +=head1 NAME + +UpdateTraitsView.pm + +=head1 SYNOPSIS + +mx-run UpdateTraitsView [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + find the correct variable_of relationship for the trait_ids in the public.traits.view + +=head1 AUTHOR + +Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package UpdateTraitsView; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the public.traits view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch updates the cvterm_relationship table and sets type_id = VARIABLE_OF from the cv = relationship +Requires running db pathc 00194 first +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Naama Menda + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package UpdateVariableOfTypeId; + +use Moose; +use Bio::Chado::Schema; +use SGN::Model::Cvterm; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates cvterm_relationship.type_id = VARIABLE_OF from the cv.name = relationship ontology + +has '+prereq' => ( + default => sub { + [], + }, +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $coderef = sub { + print STDOUT "Updating cvterm_relationship table type_id to the new variable_of cvterm \n"; + my $variable_of_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'VARIABLE_OF', 'relationship')->cvterm_id(); + my $cvterm_relationship_rs = $schema->resultset("Cv::CvtermRelationship")->search( + { + 'type.name' => 'VARIABLE_OF', + 'me.type_id' => { '!=' => $variable_of_id }, + }, + { join => 'type' } + ); + my $count = $cvterm_relationship_rs->count; + $cvterm_relationship_rs->update( { type_id => $variable_of_id } ); + print STDOUT "Updated $count rows in cvterm_relationship table to new VARIABLE_OF type_id = $variable_of_id\n"; + + return 1; + }; + try { + $schema->txn_do($coderef); + } catch { + die "Load failed! " . $_ . "\n" ; + }; + print "Done updatating cvterm_relationship!\n"; +} + + +#### +1; # +#### diff --git a/db/00196/AddVectorStockPropsToMatviewStockprop3.pm b/db/00196/AddVectorStockPropsToMatviewStockprop3.pm new file mode 100644 index 0000000000..d19743d1e6 --- /dev/null +++ b/db/00196/AddVectorStockPropsToMatviewStockprop3.pm @@ -0,0 +1,304 @@ +#!/usr/bin/env perl + +=head1 NAME + + AddVectorStockPropsToMatviewStockprop3 + +=head1 SYNOPSIS + +mx-run AddVectorStockPropsToMatviewStockprop3 [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch fixes missing 'PlantAntibioticResistantMarker' and 'BacterialResistantMarker' stock properties in the MakeRankNumberStockPropViews dbpatch (00167) + +=head1 AUTHOR + + Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddVectorStockPropsToMatviewStockprop3; + +use Moose; +use Bio::Chado::Schema; +use SGN::Model::Cvterm; + +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => ' This patch changes the rank from being stored as a string to being stored as a number.'); + +has '+prereq' => ( + default => sub { + [ ], + }, +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + # Now re-generate the materialized view (code lifted from AddVectorStockPropsToMatviewStockprop2 and MakeRankNumberStockPropViews) + my $block_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'block', 'stock_property')->cvterm_id(); + my $col_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'col_number', 'stock_property')->cvterm_id(); + my $igd_synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'igd_synonym', 'stock_property')->cvterm_id(); + my $is_a_control_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'is a control', 'stock_property')->cvterm_id(); + my $location_code_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'location_code', 'stock_property')->cvterm_id(); + my $organization_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'organization', 'stock_property')->cvterm_id(); + my $plant_index_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'plant_index_number', 'stock_property')->cvterm_id(); + my $subplot_index_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'subplot_index_number', 'stock_property')->cvterm_id(); + my $tissue_sample_index_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'tissue_sample_index_number', 'stock_property')->cvterm_id(); + my $plot_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'plot number', 'stock_property')->cvterm_id(); + my $plot_geo_json_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'plot_geo_json', 'stock_property')->cvterm_id(); + my $range_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'range', 'stock_property')->cvterm_id(); + my $replicate_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'replicate', 'stock_property')->cvterm_id(); + my $row_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'row_number', 'stock_property')->cvterm_id(); + my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); + my $T1_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'T1', 'stock_property')->cvterm_id(); + my $T2_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'T2', 'stock_property')->cvterm_id(); + my $transgenic_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'transgenic', 'stock_property')->cvterm_id(); + my $variety_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'variety', 'stock_property')->cvterm_id(); + my $notes_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'notes', 'stock_property')->cvterm_id(); + my $state_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'state', 'stock_property')->cvterm_id(); + my $accession_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession number', 'stock_property')->cvterm_id(); + my $PUI_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'PUI', 'stock_property')->cvterm_id(); + my $donor_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'donor', 'stock_property')->cvterm_id(); + my $donor_institute_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'donor institute', 'stock_property')->cvterm_id(); + my $donor_PUI_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'donor PUI', 'stock_property')->cvterm_id(); + my $seed_source_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'seed source', 'stock_property')->cvterm_id(); + my $institute_code_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'institute code', 'stock_property')->cvterm_id(); + my $institute_name_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'institute name', 'stock_property')->cvterm_id(); + my $biological_status_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'biological status of accession code', 'stock_property')->cvterm_id(); + my $country_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'country of origin', 'stock_property')->cvterm_id(); + my $germplasm_storage_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'type of germplasm storage code', 'stock_property')->cvterm_id(); + my $entry_number_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'entry number', 'stock_property')->cvterm_id(); + my $acquisition_date_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'acquisition date', 'stock_property')->cvterm_id(); + my $current_count_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'current_count', 'stock_property')->cvterm_id(); + my $current_weight_gram_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'current_weight_gram', 'stock_property')->cvterm_id(); + my $crossing_metadata_json_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'crossing_metadata_json', 'stock_property')->cvterm_id(); + my $ploidy_level_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'ploidy_level', 'stock_property')->cvterm_id(); + my $genome_structure_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'genome_structure', 'stock_property')->cvterm_id(); + my $introgression_parent_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_parent', 'stock_property')->cvterm_id(); + my $introgression_backcross_parent_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_backcross_parent', 'stock_property')->cvterm_id(); + my $introgression_map_version_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_map_version', 'stock_property')->cvterm_id(); + my $introgression_chromosome_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_chromosome', 'stock_property')->cvterm_id(); + my $introgression_start_position_bp_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_start_position_bp', 'stock_property')->cvterm_id(); + my $introgression_end_position_bp_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'introgression_end_position_bp', 'stock_property')->cvterm_id(); + my $is_blank_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'is_blank', 'stock_property')->cvterm_id(); + my $concentration_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'concentration', 'stock_property')->cvterm_id(); + my $volume_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'volume', 'stock_property')->cvterm_id(); + my $extraction_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'extraction', 'stock_property')->cvterm_id(); + my $dna_person_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'dna_person', 'stock_property')->cvterm_id(); + my $tissue_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'tissue_type', 'stock_property')->cvterm_id(); + my $ncbi_taxonomy_id_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'ncbi_taxonomy_id', 'stock_property')->cvterm_id(); + my $seedlot_quality_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'seedlot_quality', 'stock_property')->cvterm_id(); + + my $selection_marker_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'SelectionMarker', 'stock_property')->cvterm_id(); + my $cloning_organism_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'CloningOrganism', 'stock_property')->cvterm_id(); + my $cassette_name_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'CassetteName', 'stock_property')->cvterm_id(); + my $microbial_strain_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'Strain', 'stock_property')->cvterm_id(); + my $inherent_marker_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'InherentMarker', 'stock_property')->cvterm_id(); + my $backbone_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'Backbone', 'stock_property')->cvterm_id(); + my $svector_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'VectorType', 'stock_property')->cvterm_id(); + my $gene_id_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'Gene', 'stock_property')->cvterm_id(); + my $promotors_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'Promotors', 'stock_property')->cvterm_id(); + my $terminators_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'Terminators', 'stock_property')->cvterm_id(); + my $plant_marker_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'PlantAntibioticResistantMarker', 'stock_property')->cvterm_id(); + my $bacterial_maker_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'BacterialResistantMarker', 'stock_property')->cvterm_id(); + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +Add cvterms required by the vectorviewer tool. + +=head1 AUTHOR + +Lukas Mueller + +=head1 COPYRIGHT & LICENSE + +Copyright 2025 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddVectorViewerCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; + +extends 'CXGN::Metadata::Dbpatch'; + +has '+description' => ( default => <<'' ); +This patch adds cvterms for standard process interactive drone imagery + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'stock_property' => [ + 'vectorviewer_data' + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00197/AddNumberOfInsertionsCvterm.pm b/db/00197/AddNumberOfInsertionsCvterm.pm new file mode 100644 index 0000000000..23bc80e9f1 --- /dev/null +++ b/db/00197/AddNumberOfInsertionsCvterm.pm @@ -0,0 +1,73 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddNumberOfInsertionsCvterm + +=head1 SYNOPSIS + +mx-run AddNumberOfInsertionsCvterm [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds number_of_insertions stock_property cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddNumberOfInsertionsCvterm; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'number_of_insertions' stock_property cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + $schema->resultset("Cv::Cvterm")->create_with({ + name => 'number_of_insertions', + cv => 'stock_property' + }); + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00198/AddLocusGenoMarkerTable.pm b/db/00198/AddLocusGenoMarkerTable.pm new file mode 100644 index 0000000000..c8aee5d415 --- /dev/null +++ b/db/00198/AddLocusGenoMarkerTable.pm @@ -0,0 +1,87 @@ +#!/usr/bin/env perl + +=head1 NAME + +AddLocusGenoMarker + +=head1 SYNOPSIS + +mx-run AddLocusGenoMarker [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch adds the phenome.locus_geno_marker table which will be used to link a marker from an nd_protocol to +a locus (and then alleles) in the phenome.locus table. + +=head1 AUTHOR + +David Waring + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddLocusGenoMarkerTable; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; + +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch creates the phenome.locus_geno_marker table + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $coderef = sub { + $self->dbh->do(<txn_do($coderef); + } catch { + die "Load failed! " . $_ . "\n" ; + }; + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/db/00198/AddSubplotToMatview.pm b/db/00198/AddSubplotToMatview.pm new file mode 100644 index 0000000000..94a9b09b56 --- /dev/null +++ b/db/00198/AddSubplotToMatview.pm @@ -0,0 +1,1658 @@ +#!/usr/bin/env perl + + +=head1 NAME + +AddSubplotToMatview.pm + + +=head1 SYNOPSIS + +mx-run AddSubplotToMatview [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding subplot to the view + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + +package AddSubplotToMatview; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding tissue sample to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds accession ids to cvterm - support list generation with accessions ids. +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddAccessionIdCvterm; + +use Moose; +use Try::Tiny; +use Bio::Chado::Schema; +extends 'CXGN::Metadata::Dbpatch'; + +has '+description' => ( default => <<'DESC' ); +Adds the cvterm "accessions_ids" under cv "list_types", linked to dbxref "local:accessions_ids". +DESC + +has '+prereq' => ( default => sub { [] } ); + +sub patch { + my $self = shift; + + print STDOUT "\nExecuting patch: ".$self->name."\n".$self->description."\n\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + my $accession = 'accessions_ids'; + my $definition = 'accessions_ids for lists'; + my $target_db = 'local'; + my $target_cv = 'list_types'; + + try { + $schema->txn_do(sub { + + # 1) get db_id (db.name = 'local') + my $db = $schema->resultset('General::Db')->find({ name => $target_db }) + or die "DB '$target_db' not found.\n"; + my $db_id = $db->db_id; + + # 2) insert/find dbxref (db_id, accession) + my $dbxref = $schema->resultset('General::Dbxref')->find_or_create({ + db_id => $db_id, + accession => $accession, + }); + my $dbxref_id = $dbxref->dbxref_id; + + # 3) get cv_id (cv.name = 'list_types') + my $cv = $schema->resultset('Cv::Cv')->find({ name => $target_cv }) + or die "CV '$target_cv' not found.\n"; + my $cv_id = $cv->cv_id; + + # 4) insert/find cvterm + my $cvterm_rs = $schema->resultset('Cv::Cvterm'); + my $cvterm = $cvterm_rs->find({ + name => $accession, + cv_id => $cv_id, + }); + + if ($cvterm) { + # Ensure dbxref_id/definition are set as desired + my $needs_update = 0; + if ((!defined $cvterm->dbxref_id) || ($cvterm->dbxref_id != $dbxref_id)) { + $cvterm->dbxref_id($dbxref_id); + $needs_update = 1; + } + if ((!defined $cvterm->definition) || ($cvterm->definition // '') ne $definition) { + $cvterm->definition($definition); + $needs_update = 1; + } + if (!defined $cvterm->is_obsolete) { + $cvterm->is_obsolete(0); + $needs_update = 1; + } + if (!defined $cvterm->is_relationshiptype) { + $cvterm->is_relationshiptype(0); + $needs_update = 1; + } + $cvterm->update if $needs_update; + print STDERR "cvterm '$accession' already existed; updated if needed.\n"; + } else { + $cvterm = $cvterm_rs->create({ + cv_id => $cv_id, + name => $accession, + definition => $definition, + dbxref_id => $dbxref_id, + is_obsolete => 0, + is_relationshiptype => 0, + }); + print STDERR "Inserted cvterm '$accession' in cv '$target_cv'.\n"; + } + + }); + print STDOUT "You're done!\n"; + } + catch { + die "Patch failed: $_"; + }; +} + +1; diff --git a/db/00199/AddAccessionIdSearch.pm b/db/00199/AddAccessionIdSearch.pm new file mode 100644 index 0000000000..eff14b38cf --- /dev/null +++ b/db/00199/AddAccessionIdSearch.pm @@ -0,0 +1,1823 @@ +#!/usr/bin/env perl + + +=head1 NAME + +AddAccessionIdSearch.pm + + +=head1 SYNOPSIS + +mx-run AddAccessionIdSearch [options] -H hostname -D dbname -u username [-F] + +this is a subclass of L +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding subplot to the view + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + +package AddAccessionIdSearch; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding tissue sample to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION + +This patch: + - updates the materialized_phenoview by adding subplot to the view + +=head1 AUTHOR + +Chris Simoes + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + +package FixTissueSampleSearch; + +use Moose; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch updates the materialized_phenoview by adding tissue sample to the view + + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + + $self->dbh->do(< +see the perldoc of parent class for more details. + +=head1 DESCRIPTION +This patch adds 'derived_from' stock_relationship cvterm +This subclass uses L. The parent class uses L + +=head1 AUTHOR + +Titima Tantikanjana + +=head1 COPYRIGHT & LICENSE + +Copyright 2010 Boyce Thompson Institute for Plant Research + +This program is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut + + +package AddDerivedAccessionRelatedCvterms; + +use Moose; +use Bio::Chado::Schema; +use Try::Tiny; +extends 'CXGN::Metadata::Dbpatch'; + + +has '+description' => ( default => <<'' ); +This patch adds the 'derived_from' stock_relationship cvterm + +has '+prereq' => ( + default => sub { + [], + }, + +); + +sub patch { + my $self=shift; + + print STDOUT "Executing the patch:\n " . $self->name . ".\n\nDescription:\n ". $self->description . ".\n\nExecuted by:\n " . $self->username . " ."; + + print STDOUT "\nChecking if this db_patch was executed before or if previous db_patches have been executed.\n"; + + print STDOUT "\nExecuting the SQL commands.\n"; + my $schema = Bio::Chado::Schema->connect( sub { $self->dbh->clone } ); + + print STDERR "INSERTING CV TERMS...\n"; + + my $terms = { + 'stock_relationship' => [ + 'derived_from', + ], + }; + + foreach my $t (keys %$terms){ + foreach (@{$terms->{$t}}){ + $schema->resultset("Cv::Cvterm")->create_with({ + name => $_, + cv => $t + }); + } + } + + print "You're done!\n"; +} + + +#### +1; # +#### diff --git a/docs/404.html b/docs/404.html new file mode 100644 index 0000000000..91f8a5a1f5 --- /dev/null +++ b/docs/404.html @@ -0,0 +1,459 @@ + + + + + + + Page not found | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Page not found

+

The page you requested cannot be found (perhaps it was moved or renamed).

+

You may want to try searching to find the page's new location, or use +the table of contents to find the page you are looking for.

+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/BreedbaseManual.pdf b/docs/BreedbaseManual.pdf new file mode 100644 index 0000000000..ea0c125d63 Binary files /dev/null and b/docs/BreedbaseManual.pdf differ diff --git a/docs/BreedbaseManual.tex b/docs/BreedbaseManual.tex new file mode 100644 index 0000000000..df331fd8bc --- /dev/null +++ b/docs/BreedbaseManual.tex @@ -0,0 +1,3547 @@ +% Options for packages loaded elsewhere +\PassOptionsToPackage{unicode}{hyperref} +\PassOptionsToPackage{hyphens}{url} +\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor} +% +\documentclass[ + 12pt, +]{book} +\usepackage{amsmath,amssymb} +\usepackage{iftex} +\ifPDFTeX + \usepackage[T1]{fontenc} + \usepackage[utf8]{inputenc} + \usepackage{textcomp} % provide euro and other symbols +\else % if luatex or xetex + \usepackage{unicode-math} % this also loads fontspec + \defaultfontfeatures{Scale=MatchLowercase} + \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1} +\fi +\usepackage{lmodern} +\ifPDFTeX\else + % xetex/luatex font selection +\fi +% Use upquote if available, for straight quotes in verbatim environments +\IfFileExists{upquote.sty}{\usepackage{upquote}}{} +\IfFileExists{microtype.sty}{% use microtype if available + \usepackage[]{microtype} + \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts +}{} +\makeatletter +\@ifundefined{KOMAClassName}{% if non-KOMA class + \IfFileExists{parskip.sty}{% + \usepackage{parskip} + }{% else + \setlength{\parindent}{0pt} + \setlength{\parskip}{6pt plus 2pt minus 1pt}} +}{% if KOMA class + \KOMAoptions{parskip=half}} +\makeatother +\usepackage{xcolor} +\usepackage{longtable,booktabs,array} +\usepackage{calc} % for calculating minipage widths +% Correct order of tables after \paragraph or \subparagraph +\usepackage{etoolbox} +\makeatletter +\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{} +\makeatother +% Allow footnotes in longtable head/foot +\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}} +\makesavenoteenv{longtable} +\usepackage{graphicx} +\makeatletter +\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi} +\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi} +\makeatother +% Scale images if necessary, so that they will not overflow the page +% margins by default, and it is still possible to overwrite the defaults +% using explicit options in \includegraphics[width, height, ...]{} +\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio} +% Set default figure placement to htbp +\makeatletter +\def\fps@figure{htbp} +\makeatother +\setlength{\emergencystretch}{3em} % prevent overfull lines +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\setcounter{secnumdepth}{5} +\ifLuaTeX + \usepackage{selnolig} % disable illegal ligatures +\fi +\usepackage[]{natbib} +\bibliographystyle{plainnat} +\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}} +\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available +\urlstyle{same} +\hypersetup{ + pdftitle={Breedbase User Manual}, + pdfauthor={Breedbase team}, + colorlinks=true, + linkcolor={Maroon}, + filecolor={Maroon}, + citecolor={Blue}, + urlcolor={Blue}, + pdfcreator={LaTeX via pandoc}} + +\title{Breedbase User Manual} +\author{Breedbase team} +\date{2026-01-08} + +\begin{document} +\maketitle + +{ +\hypersetup{linkcolor=} +\setcounter{tocdepth}{2} +\tableofcontents +} +\hypertarget{introduction}{% +\chapter*{Introduction}\label{introduction}} + + +Welcome to the Breedbase manual! + +This manual is intended for database users.\\ +If you are a developer looking for software implementation details, please visit the developer wiki instead: \url{https://github.com/solgenomics/sgn/wiki} + +\hypertarget{basic-website-usage}{% +\chapter{Basic Website Usage}\label{basic-website-usage}} + +Breedbase is usually hosted on the cloud and is entirely web-based, with only a browser required to access and use it. The recommended browser is Firefox. To use Breedbase, you or your project will usually need to have your own Breedbase instance. Information how you can obtain your own instance can be obtained from the Breedbase project (\url{https://breedbase.org/}). + +Once an instance is set up, the site needs to be configured and some metadata uploaded before you can design, run and analyze trials: + +\begin{itemize} +\item + define user accounts and their access privileges. +\item + add a trait ontology with the traits that you require. +\item + add the locations that you use in your breeding program +\item + add the foundational germplasm and the respective pedigree data +\item + ``historical'' trial data can be uploaded as needed. Usually more recent trials are prioritized over older trials +\end{itemize} + +In this chapter, we will cover how you can manage the user accounts and some basic website features such as lists and how to navigate the menus. The other topics are covered in subsequent chapters. Note that Breedbase instances are highly customizable, and that not every instance of Breedbase will have the same options in the same location. Refer to the site specific documentation, if any, for site specific information. + +\hypertarget{creating-a-user-account}{% +\section{Creating a User Account}\label{creating-a-user-account}} + +\hypertarget{verifying-first-that-you-do-not-already-have-an-account}{% +\subsection{Verifying first that you do not already have an account}\label{verifying-first-that-you-do-not-already-have-an-account}} + +Before creating an account, please verify first that you don't already have an account. You can use ``Search'' menu to check if you already registered as a user. + +In the ``Search'' menu, selecting the ``People'' tab and search your name. If nothing is found, proceed with the instructions below. Otherwise, continue by clicking the ``Login'' button. If you have forgotten your password, you can retrieve it by clicking the ``Forgot your password?'' link on the login page. + +\hypertarget{creating-a-user-account-1}{% +\subsection{Creating a user account}\label{creating-a-user-account-1}} + +On the right of the toolbar, click on ``Login'', which will take you to the login dialog. On the login dialog, click on the link ``sign up for an account.'' It will take you to the page below: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image331} \end{center} + +Filling in all of the information, then click ``Create Account.'' + +After you submit the information, an email will be sent to the provided email address. Check your email and click on the link to activate your account. + +\hypertarget{managing-your-account}{% +\section{Managing your Account}\label{managing-your-account}} + +\hypertarget{login}{% +\subsection{Login}\label{login}} + +To login, click the ``Login'' link in the toolbar on any page and enter your username and password. + +If you have forgotten your password, you can retrieve it by clicking the ``Forgot your password?'' link on the login page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image166} \end{center} + +\hypertarget{editing-account-settings}{% +\subsection{Editing Account Settings}\label{editing-account-settings}} + +Account settings can be edited by clicking on the ``my profile'' link displayed as your user name, on the right of the toolbar. You must be logged in order to access and change account settings. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image261} \end{center} + +You can add personal information to your account using the ``View or update personal information'' link. + +To change your password, username, or your contact email, click on ``Update account information'' link. You must provide your old password before you can make any changes. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image144} \end{center} + +\hypertarget{changing-your-account-status-from-user-to-submitter}{% +\subsection{Changing Your Account Status: From ``User'' to ``Submitter''}\label{changing-your-account-status-from-user-to-submitter}} + +After you create an account, your account has a ``user'' status. This account has limited privileges. + +Accounts with ``user'' status are able to: + +\begin{itemize} +\tightlist +\item + Change personal information +\item + Post comments on pages +\item + Post to the forum +\end{itemize} + +To upgrade your account status to ``submitter,'' contact the database curators using the ``contact'' link provided at the footer of each page. Submitter accounts can add data, such as new plots, accessions, phenotypic data and images. + +\hypertarget{submitting-feedback-on-an-sgn-database}{% +\subsection{Submitting Feedback on an SGN Database}\label{submitting-feedback-on-an-sgn-database}} + +We appreciate your feedback! Feel free to submit any questions or suggestions by using the ``Feedback'' link provided at the footer of each page. + +\hypertarget{menu-layout}{% +\section{Menu Layout}\label{menu-layout}} + +SGN Database websites have a toolbar on the top of each page with a number of menus for convenient access of major functions. The menus, as pictured below, are ``search,'' ``manage,'' ``analyze,'' and ``maps.'' The toolbar also provides a quick search, a ``log in'' button, and a ``new user'' button. The menus can be customized for each instance, so they may not appear exactly as shown on your instance; for example, most instances do not have a map menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image270} \end{center} + +\hypertarget{menu-options}{% +\subsection{Menu Options}\label{menu-options}} + +\hypertarget{search}{% +\subsubsection*{Search}\label{search}} + + +In the Search menu, the options are: + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.2083}} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.7917}}@{}} +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Tab +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +Description +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +Wizard & Search different accessions and plots by location, year, trial, and trait data. Can also be used to create lists of different types. \\ +Accession and plots & Search accessions and plots using a variety of criteria \\ +Trials & Search trials by name, description, breeding program, year, location, and trial type. \\ +Markers & Search different markers \\ +Images & Search images contained in the SGN database \\ +People & Search database users \\ +\end{longtable} + +\hypertarget{manage}{% +\subsubsection*{Manage}\label{manage}} + + +In the Manage menu, the options are: + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.2222}} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.7778}}@{}} +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Tab +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +Description +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +Breeding Programs & View, add and delete breeding programs \\ +Locations & View, add and delete locations \\ +Accessions & Manage and search different accessions \\ +Seedlots & Manage and search different seedlots \\ +Crosses & Create new crosses in the database \\ +Field Trials & Manage field trials. Create trials using different field layouts. \\ +Genotyping Plates & Manage genotyping plates. Create 96 or 384 well plates. \\ +Phenotyping & Upload phenotyping files from the Tablet Field Book application \\ +Field Book App & Manage the field book app data (download files to tablet) \\ +Barcodes & Refers to the old barcode system, mainly historical \\ +Download & Download information in the database based on lists \\ +\end{longtable} + +\hypertarget{analyze}{% +\subsubsection*{Analyze}\label{analyze}} + + +\textbf{Clicking on the ``Analyze'' link will give a full menu of all analysis functions}\\ +In the Analyze menu, the options are: + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.2361}} + >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.7639}}@{}} +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Tab +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +Description +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +\textbf{Breeder Tools} & \\ +Breeder Home & Access breeding functionalities. Lists important and helpful links. \\ +Barcode Tools & Manage, create, and download barcodes. Also access barcode tools. \\ +Genomic Selection & Can search for traits, start building a GS model, and predict values based on genotypes \\ +\textbf{Sequence Analysis} & \\ +BLAST & Sequence homology search \\ +\textbf{Other} & \\ +Ontology Browser & Browse all recorded ontologies \\ +\end{longtable} + +\hypertarget{working-with-lists}{% +\section{Working with Lists}\label{working-with-lists}} + +Lists are collections of identifiers that are stored in the database. Lists can be composed of accessions, plots, traits, locations, and trials. A given list can only contain one type of items, for example, all items have to be of type location in a list of locations. Lists are attached to the individual user's account, and can only be created and seen by the user while logged in, however, lists can be made public for all other users to see. SGN databases make heavy use of lists in a number of tools on the website. For example, trials are created using lists of accessions. + +\hypertarget{creating-lists}{% +\subsection{Creating lists}\label{creating-lists}} + +Lists can be generated in various ways: + +One way to create a list is by clicking on the ``Lists'' link located on the toolbar. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_start} \end{center} + +To create a new list, enter the name of your new list and then click on the ``New List'' button. The name of the list can be anything, but should be unique and should be something to help you easily identify the list. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_new_list} \end{center} + +If the list already exists, it will appear on the ``Your Lists'' dialog. To add items to your list, click on the ``View'' icon to open the ``List Contents'' section. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_view_list} \end{center} + +On the ``List Contents'' page, enter items that you want to add to the list, then click on ``Add'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_add_items} \end{center} + +The page will be updated and will display your items in a table at the bottom of the dialog It is possible to sort the list if you need. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_added_items} \end{center} + +Select the type of items in your list. To verify that the items that you added to your list are already stored in the database and that you selected a correct type for the items, click on the ``Validate'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/list_manager_list_types} \end{center} + +If those items are already in the database, a message will indicate that ``This list passed validation'' + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image346} \end{center} + +Note that a list cannot contain duplicate elements. If a duplicate item is entered, the list manager will inform the user that the element is already in the list and will not add it again. + +Another easy way to create a list is to use the \ref{search-wizard}, which can be accessed from the Search menu. + +\hypertarget{viewing-and-editing-lists}{% +\subsection{Viewing and editing lists}\label{viewing-and-editing-lists}} + +Lists can be viewed and edited using the ``Lists'' link on the toolbar. Clicking on the link will open a dialog that displays all of your lists, as well as an option to create new lists. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image71} \end{center} + +This page shows all lists that have been created, including those created by using the Search Wizard. You can view and edit your lists by using ``Actions'' buttons. + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\item + Clicking on the ``view'' icon will open a new window called ``List Contents'' that allows you to change the list name, the type of the list, add new items, or delete existing items. +\item + Clicking on the ``delete'' icon will delete your list. \textbf{Caution: this action cannot be undone}. +\item + Clicking on the ``download'' icon will download the contents of your list to your computer. +\item + Clicking on the ``make public'' icon will make your list available for other users to view and use your list. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image297} \end{center} + +\hypertarget{user-permissions}{% +\section{User Permissions}\label{user-permissions}} + +Breedbase accounts are assigned one or more of four different roles to determine the level of access they have within the database. The possible roles are \textbf{User}, \textbf{Submitter}, \textbf{Sequencer}, and \textbf{Curator}. Each role grants specific permissions, and careful management of them helps prevent data from being altered or deleted in error. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/roles} \end{center} + +Accounts are also assigned Breeding Program role(s) to grant access to the specfic breeding program(s) they work with. + +\begin{itemize} +\tightlist +\item + The \textbf{User} role gives an account permission to view and download data throughout the database. +\item + The \textbf{Submitter} role gives an account permission to design field experiments and to upload and edit data using the tools in the ``Manage'' section. In order to submit and manage breeding data within a given breeding program, a submitter also must have a matching Breeding Program role. +\item + The \textbf{Sequencer} role gives an account permission to design genotyping experiments and submit plates to a genotyping service. +\item + The \textbf{Curator} role gives an account permission to do all of the above, as well as to delete data within the database. The Curator role also enables the addition or deletion of roles for all database accounts in the `Manage User Roles' tool. +\end{itemize} + +\hypertarget{searching-the-database}{% +\chapter{Searching the Database}\label{searching-the-database}} + +You can search for information on the database by using the following search options: Wizard, which uses combined criteria specified by users; Accessions and Plots; Trials; Markers; Images; People; FAQ. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image267} \end{center} + +\hypertarget{search-wizard}{% +\section{The Search Wizard}\label{search-wizard}} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/wizard_interface} \end{center} + +\hypertarget{how-the-search-wizard-works}{% +\subsection{How the Search Wizard Works}\label{how-the-search-wizard-works}} + +The search wizard presents a number of select boxes, which are initially empty. You start searching by picking a category of data from the dropdown above the left-most select box. + +Once a category has been picked, the database will retrieve all the options within this category and display them within the first select box. You then select one or more options from the first select box, which activates the second dropdown. + +You can then select a category from the second dropdown, and repeat this same search process through all four dropdowns and select boxes. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/wizard_interface_selections} \end{center} + +\begin{itemize} +\item + In the example above, the ``locations'' category was chosen in the first dropdown. The first select box then displayed all the possible locations in the database. The option Ibadan was selected. +\item + This activated the second dropdown. The category ``years'' was chosen in the second dropdown. The second select box then displayed all the years that are linked in the database to the location Ibadan. From that list, the options 2011 and 2012 were selected. +\item + This activated the third dropdown. A final category, ``accessions'', was chosen in the third dropdown. The third select box was then populated with the 3847 accessions in the database that are linked with the location Ibadan in the years 2011 or 2012. +\end{itemize} + +In addition to the basic search operations demonstrated above, users can take advantage of two more features: + +\textbf{Load Selection from List} + +\begin{center}\includegraphics[width=0.25\linewidth]{assets/images/wizard_select_list} \end{center} + +\begin{itemize} +\tightlist +\item + Instead of picking a category in the first dropdown, users can instead populate the first selectbox from a list by scrolling down in the first dropdown to the ``Load Selection from List'' subheading and selecting a list. This is useful for starting queries with a list of plots, as this category is not among the options in the first dropdown. +\end{itemize} + +\textbf{ANY/MIN/ALL} Toggle + +\begin{center}\includegraphics[width=0.25\linewidth]{assets/images/wizard_any_min_all_toggle} \end{center} + +\begin{itemize} +\item + By default, the search wizard combines options within a category using an OR query. In the example above, in the third panel the wizard retrieved accessions associated with the location `Ibadan' in \textbf{ANY} of the years ``2011 \textbf{OR} 2012'' +\item + If the user clicked the toggle below the second select box to change it to \textbf{ALL} before choosing accessions in the third dropdown, the wizard would instead retrieve accessions associated with the location `Ibadan' in the years ``2011 \textbf{AND} 2012''. This will be a smaller set of accessions, because any accessions used only in 2011, or only in 2012 will be excluded. +\item + A more advanced search could use the \textbf{MIN} toggle option. This allows the user to make a query in between an ANY or ALL query, where a minimum number of matches from the selected column will be used as a filter for the next column. The minimum can be provided as either a percentage (\%) or an actual count of items (\#). In the example above, if the years 2011, 2012, and 2013 were selected in the second column, the user could enter `2' in as the minimum and select `\#' as the minimum match type. This would select accessions in the third column that were used in 2 or more of the selected years. +\end{itemize} + +\begin{center}\includegraphics[width=0.25\linewidth]{assets/images/wizard_any_min_all_toggle_min_details} \end{center} + +\hypertarget{how-to-use-retrieved-data}{% +\subsection{How to use retrieved data}\label{how-to-use-retrieved-data}} + +\hypertarget{getting-more-info}{% +\subsubsection*{Getting more Info}\label{getting-more-info}} + + +Any option in the wizard select boxes (except for years) can be clicked to open a page with more details. The new page is opened in a new tab. + +\hypertarget{saving-to-a-list}{% +\subsubsection*{Saving to a list}\label{saving-to-a-list}} + + +You can store the highlighted items in any selected box to lists. This is done using the inputs and buttons directly below the select box. \textbf{Don't forget, you must be logged in to work with lists!} + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/add_create_list} \end{center} + +\begin{itemize} +\item + To \textbf{add items to an existing list}, first pick an existing list using the ``Add to List\ldots{}'' dropdown on the left. Then click the ``Add'' button. A popup window will confirm the action, and display the number of items added to your existing list. +\item + To \textbf{store items to a new list}, first type a new list name in the ``Create New List\ldots{}'' text input on the left. Then click on the ``Create'' button. A popup window will confirm the action, and display the number of items added to your new list. +\end{itemize} + +\hypertarget{downloading-data}{% +\subsubsection*{Downloading Data}\label{downloading-data}} + + +You can download trial metadata, phenotypes and genotypes associated with the highlighted items in the wizard select boxes. This is done using the buttons in the download section at the bottom of the page. \textbf{Don't forget, you must be logged in to download data!} + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/wizard_download_options} \end{center} + +\hypertarget{metadata}{% +\paragraph*{Metadata}\label{metadata}} +\addcontentsline{toc}{paragraph}{Metadata} + +Trial metadata can be downloaded by selecting a subset of trials from the database or based on your search categories. To download, click on ``Related Trial Metadata'', a dialog will appear. Select download format and click the ``Metadata'' button to complete your download. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/wizard_related_metadata_download} \end{center} + +\hypertarget{phenotypes}{% +\paragraph*{Phenotypes}\label{phenotypes}} +\addcontentsline{toc}{paragraph}{Phenotypes} + +The phenotypes download is quite flexible, and can download a subset of all the trial data in the database based on whichever categories and options you currently have selected. Simply click on the ``Related Trial Phenotypes'' link, review the options, changing or adding any additional parameters you like, then click `Download Phenotypes'. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/wizard_related_phenotypes_download} \end{center} + +\hypertarget{genotypes}{% +\paragraph*{Genotypes}\label{genotypes}} +\addcontentsline{toc}{paragraph}{Genotypes} + +The genotype download is more stringent. It requires a minimum of one accession and one genotyping protocol to be selected in the wizard select boxes. The text box in the download section of the page will help track what has been selected. Once clicked, the ``Download Genotypes'' button will download a genotype file for the selected accessions. + +\hypertarget{saving-the-wizard-selections}{% +\subsubsection*{Saving the wizard selections}\label{saving-the-wizard-selections}} + + +As discussed above, the selections of the individual select boxes in the wizard can be saved separately to a list. The lists can be used as inputs in other tools on the site. However, sometimes creating a selection is quite time consuming and restoring the selections from four different lists would be cumbersome too. Therefore, the selections can be saved together in a dataset, and named for later retrieval. This is done in the section ``Load/Create Datasets'' that is below the first two wizard select boxes. To select an existing dataset, one uses the ``Load Dataset'' dropdown. A particular dataset can be chosen, and the ``Load'' button can be clicked to retrieve and display the dataset in the wizard. To create a new dataset using items that are selected in the wizard, one can enter the name of the new dataset in the ``Create New Dataset'' text box. Once the dataset has been given a name, clicking the ``Create'' button will save the new dataset. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/wizard_load_create_dataset} \end{center} + +\hypertarget{updating-the-wizard}{% +\subsection{Updating the Wizard}\label{updating-the-wizard}} + +The search wizard uses a copy of the database, or a cache, to return results quickly. If data appears to be missing, it usually means that the cache needs to be updated. Users with submitter privileges or above can do this using the `Update Wizard' button. One can also use the `Refresh Lists' button to update the available lists. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/wizard_update_list_refresh} \end{center} + +This will take just a few seconds in small databases, but may take a few hours to complete in larger databases. + +\hypertarget{accessions-and-plot-search}{% +\section{Accessions and Plot Search}\label{accessions-and-plot-search}} + +Accessions and their related materials (cross, plant, plot, population, tissue\_sample, training population) can be searched by using ``Search Accessions and Plots'' page. On this page, ``accession'' is the default stock type; however, you can change stock type by selecting an option from the drop-down list. From this page you can construct detailed queries for stock types. For example, by using the ``Usage'' section, the ``Properties'' section, and the ``Phenotypes'' section you could search for accessions which were diploids used in a specific year and location and were also phenotyped for height. You can also search for accessions based on genetic properties, such as the location of an introgression on a specific chromosome. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_accessions} \end{center} + +It is possible to query over any of the available properties, such as ``ploidy\_level'', ``country of origin'', ``introgression\_chromosome'', etc. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_accessions_properties_search} \end{center} + +In the search result table it is possible to select any of the available properties to view. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_accessions_properties_view} \end{center} + +At the bottom of the accession search there is a phenotype graphical filtering tool. Here you can filter down accessions based on combinations of trait performance. The filtered down accessions are then able to be saved to a list. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_accessions_graphical_filtering} \end{center} + +For information on adding Accessions please see the Managing Accessions help. For information on how field trial plots, plants, tissue samples, and subplots are added to the database, please see the Managing Field Trials help. + +\hypertarget{trials-search}{% +\section{Trials Search}\label{trials-search}} + +Trials on the database can be searched based on trial name, description, breeding program, year, location, trial type, design, planting date, and harvest date. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_search} \end{center} + +\hypertarget{trait-search}{% +\section{Trait Search}\label{trait-search}} + +On the Trait Search page (menu item \texttt{Search\ \textgreater{}\ Traits}), traits in the database can be searched by ID, name, or descripiton. Optionally, a starting list of traits can be selected to filter down results. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trait-search-default} \end{center} + +Selecting traits in the results of the search allows one to add the selected results to a trait list, or create a new trait list from the select results. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trait-search} \end{center} + +\hypertarget{ontology-browser}{% +\section{Ontology Browser}\label{ontology-browser}} + +A more advanced tool for searching for Traits is the ontology browser, available by clicking on Analyze and Ontology Browser. From here you can search ontologies and see the various classifications of terms in a tree display. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/ontology_browser} \end{center} + +The terms which appear in the Trait Search in 2.4 are only variable terms. The ontology browser shows these variables as different from their grouping terms by indicating VARIABLE\_OF like in the following screenshot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/ontology_browser_variable} \end{center} + +\hypertarget{search-seedlots}{% +\section{Search Seedlots}\label{search-seedlots}} + +Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. To search for available seedlots you go to Manage and then click Seed Lots. By clicking Search Seedlots, you can specify query information. The results from your search will be in the table below the search form. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_seedlots} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlots} \end{center} + +\hypertarget{managing-user-roles}{% +\chapter{Managing User Roles}\label{managing-user-roles}} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_user_roles_page} \end{center} + +\hypertarget{what-are-user-roles}{% +\section{What are User Roles?}\label{what-are-user-roles}} + +Every user account in Breedbase has one or more associated ``roles'' that determine the authorizations (what the user is allowed to do) in the database. There are three fundamental roles, ``curator'', ``submitter'', and ``user'', which determine basic read/write levels. The ``curator'' status can read and write everything in the database. The ``submitter'' status can add information and edit or delete previously submitted information. The ``user'' type can only read data. Additional roles represent the breeding programs, and are sometimes used to fine-tune write and edit capabilities, as it necessary for multiple users in a breeding program to edit each other's data. + +\hypertarget{the-manage-user-roles-page}{% +\section{The Manage User Roles page}\label{the-manage-user-roles-page}} + +In the ``Manage'' menu, select the item ``User Roles''. This will show the current users in the database with their associated roles. If you are logged in as a curator, the table will show system roles as well as breeding program roles; if you are logged in as a submitter or user, it will show breeding program membership. + +If logged in as a ``curator'', the roles can be added or deleted. + +\begin{itemize} +\tightlist +\item + To delete a role, click on the X in the role name. A confirm dialog will be displayed to prevent accidental deletion. +\item + To add a role, click on the plus sign next to the roles. A dialog will pop up with a list of roles. Select the desired role and click ``Submit''. +\item + The new role should be displayed next to the user immediately. +\item + Role deletions and additions will be effective immediately. +\end{itemize} + +It is recommended that few users be given the ``curator'' privileges to avoid confusion over data ownership and accidental data overwriting and deletion. + +\hypertarget{managing-breeding-programs}{% +\chapter{Managing Breeding Programs}\label{managing-breeding-programs}} + +New breeding programs can be added by using ``Add New Program'' button on the ``Manage Breeding Programs'' page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/breeding_programs_screenshot} \end{center} + +Clicking on the ``Add New Program'' button will generate a blank form for you to fill out the name and description of the breeding program that you want to add. After completing the form, click on ``Add Breeding Program'' button to finish the process. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/new_program_screenshot} \end{center} + +\hypertarget{managing-locations}{% +\chapter{Managing Locations}\label{managing-locations}} + +Field locations can be managed using the ``\textbf{Manage Locations}'' page. On this page, locations in the database are organized based on their breeding programs. Each location has a link to trials conducted in that location. To add a new location, click on the ``Upload New Locations'' button that links to the ``Upload Locations'' form. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image141} \end{center} + +The ``Upload Locations'' describes how to build a spreadsheet with location data for upload. Name, abbreviation, country code, country name, program, type, latitute, longitude, and elevation are all required. The NOAA station ID is optional. Link a spreadhsheet to the form and click ``Upload'' to add those locations to the database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image187} \end{center} + +Alternatively, locations can be viewed and added via the map. Hover over an icon on the map to see the location details and trials linked to that location. Click on the map to open the new location dialog. Fill in the same information that would be used in the spreadsheet upload to add a new location. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/location_map} \end{center} + +\hypertarget{managing-accessions}{% +\chapter{Managing Accessions}\label{managing-accessions}} + +The ``Manage Accession'' page provides links for adding new accessions. New accessions can be added to the database by either using a List or by uploading an Excel file (either XLS or XLSX format). Both options are explained in more detail below. To begin, click on the ``Add Accessions or Upload Accession Info'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_link} \end{center} + +This will open a dialog allowing you to select either ``Using Lists'' or ``Uploading a File''. + +\hypertarget{add-accessions-using-a-list}{% +\section{Add Accessions Using A List}\label{add-accessions-using-a-list}} + +First we will show how to add accessions ``Using Lists''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_using_lists} \end{center} + +Here you select an accession list which you have previously made (see List Manager chapter). If you need to create or edit your list you can do so now by clicking ``Manage Lists''. After selecting your list, click ``Continue''. + +The contents of the list will be checked against the database, and elements that are already present will be flagged. A dialog will appear that will show the accessions which already exist in the database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_found} \end{center} + +After clicking on the ``Continue'' button, the next dialog will appear with accessions that have very similar names as the accession that you are adding. In the example below, there are two accession with very similar names to accessions already in the database. `TME0419' is very similar to `TME419', and probably represent the same line, so it would be a mistake to add this the database again. Duplicate lines in the database should be avoided, as they cause problems when evaluating lines; data is divided up among several duplicates, making it harder to get the full picture about an accession. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_fuzzy} \end{center} + +To avoid situations in adding a mistaken duplicate accession, the database gives you options for moving forward with these very similar looking accession names. You can either ``continue saving the name in your list'', ``replace name in your list with selected existing name'', ``remove name in your list and ignore'', or ``add name in your list as a synonym to selected existing name''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_fuzzy_options} \end{center} + +Clicking ``Download Fuzzy Matches'' will return a tabular result of the ``fuzzy'' accession name results shown. Click ``Make changes and continue'' to move on. + +The final dialog shows the accessions that will be added. Here you need to assign the species of these accessions. You can optionally group the accessions into a population and/or add an organization for the accessions. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_complete_using_list} \end{center} + +Once you click ``Add Accessions'', the new accessions will be created in the database and you will see the following confirmation dialog, which includes links to the newly created accessions. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_saved} \end{center} + +\hypertarget{uploading-accessions-and-accessions-info-from-a-file}{% +\section{Uploading Accessions and Accession's Info From A File}\label{uploading-accessions-and-accessions-info-from-a-file}} + +Uploading accessions using a file is very similar to using a list, but enables you to add a variety of attributes, such as synonyms or ploidy levels, to the accessions in bulk. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_using_file} \end{center} + +Clicking on ``Spreadsheet format'' will show the required structure of the spreadsheet. The file must be XLS or XLSX format and can contain a number of header columns as attributes. It is important that you use exactly the same header column names as listed here. In columns that indicate that many attribute values can be passed at once using (s), such as synonym(s), you can pass a comma separated list of values, such as `synonym1,synonym2'. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_spreadsheet_info} \end{center} + +Once you have selected your XLS or XLSX file for upload, click ``Continue''. + +The following process is the same way as with lists: + +The first dialog which can appear will show accession names which are already in the database. + +Click ``Continue'' and the next dialog that can appear will show ``fuzzy'' matches for the accession names you are trying to upload. Here you can choose to prevent adding accession names which look very similar to each other as wrongly duplicated accessions. + +Click ``Continue'' and the final dialog that will appear will show the information to be added into the database. Here it is divided into accession names that are new and accession names that already exist in the database; however, for the accession names that already exist it will show additional attributes that originated from your file that will be added to these accessions. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_accessions_add_accessions_complete_using_file} \end{center} + +Once you click ``Add Accessions'', the new accessions and information will be created in the database and you will see the following confirmation dialog, which includes links to the created and updated accessions. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/manage_accessions_add_accessions_saved} \end{center} + +\hypertarget{email-alert-for-accession-upload}{% +\section{Email alert for accession upload}\label{email-alert-for-accession-upload}} + +When uploading a large number of accessions from a file, uploads can take a while, as the system needs to perform a series of checks on each entry. You have the option to receive an email notification about the status and results of your upload by clicking the ``Email Alert'' checkbox. By default, the system will use the email address associated with your account, but you have the option of entering a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results, including any warnings or errors that may have occurred during the upload. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/accession_upload_using_email} \end{center} + +\hypertarget{add-parentage-pedigree-information-to-accessions}{% +\section{Add Parentage (Pedigree) Information to Accessions}\label{add-parentage-pedigree-information-to-accessions}} + +Pedigree data can be uploaded from your computer by clicking on ``Upload Pedigree File'' + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image286} \end{center} + +You can find detailed information on how to prepare pedigree file by clicking on ``File format information''. The file format can be tab or comma delimited text file, or Excel files (.xls or .xlsx). + +The currently supported format has four columns: + +progeny name female parent accession male parent accession type + +Type can be biparental, self, backcross, sib, polycross, reselected, or open. In the case of the open type, the male parent accession field can remain blank. For all other types, both columns should be filled, even if they contain the same information as another column (such as self). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image333} \end{center} + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/pedigree_upload_format} \end{center} + +\hypertarget{working-with-grafts}{% +\section{Working with grafts}\label{working-with-grafts}} + +Grafts are plants that are composed of a rootstock and a scion, which are genetically different and fused together, usually at the stem level. + +To work with grafts, the grafts interface needs to be activated by a system administrator. Please contact your Breedbase provider. Briefly, a configuration parameter needs to be added to the sgn\_local.conf file, show\_grafting\_interface. It should be set to 1 in sgn\_local.conf, the default is 0 in sgn.conf. + +Grafts to be created need to be specified using an Excel file (xlsx format) with two columns. The first column should have the header ``scion accession'' and should list accession names that will be scions. The second column should have the header ``rootstock accession'' and should list accession names that will be rootstocks. + +In the database, the graft accessions will be created as single accessions. The graft accession will have two relationships, one to the scion accession (scion\_of relationship) and one to the rootstock (rootstock\_of relationship). These relationships are displayed on the pedigree viewer. The graft accession name is created from the scion accession name and the rootstock accession name, separated by the graft separator character. By default, the graft separator character is the plus sign `+'. The graft separator character can be changed in the sgn\_local.conf file, using the parameter graft\_separator\_string. The graft separator string should not occur in any other accession names that are not grafts. + +When the grafting interface is activated, a new button will be shown on the manage accessions page, called ``Upload Grafts''. + +Clicking the button brings up the upload grafts dialog. + +Select the Excel file containing the grafting information. The system will validate the file, for example, check whether the accessions are in the database, and if the headers are correct. + +The validation result will be presented, and if problems are found, they will be listed. In addition, if there are problems, the Upload button will be grayed out and upload will not be possible. Conversely, if there are no problems, the Upload button will be activated and can be clicked to store the data. + +If the upload completes, a completion message is displayed with a summary what was uploaded. + +Grafted accessions can be used like any other accession, for example, they can be used on field layouts. If you create a list of graft accessions, use the list type `accessions'. + +Note that you shouldn't create new grafts based on other grafts. The scion accession and the rootstock accession have to be different, otherwise they will not be created. + +\hypertarget{bulk-renaming-of-accessions}{% +\section{Bulk renaming of accessions}\label{bulk-renaming-of-accessions}} + +Accessions can be renamed in bulk using the rename accessions feature. To rename accessions, prepare a tab delimited file with two columns: the first column should have the header ``old name'' and contain the accession names that need to be changed. The second column should have the header ``new name'' and contain the names that the accessions in column 1 should be renamed to. + +The accession renaming feature is available from the Manage-\textgreater Accessions page. Click on the ``Rename Accessions'' button. The first step is the upload of the file with a verification step. The verification step checks whether all the accession names in column 1 exist in the database, and whether all the accession names given in column 2 do NOT exist in the database. Only if both conditions are met, will the ``rename'' button become active, otherwise an error message is displayed listing the offending accession names. + +Optionally, the old name can be automatically added as a synonym to the renamed accession, using the checkbox on the submit form. This option is clicked by default. Unclick the checkbox to NOT save any old names as synonyms. + +Note that accession renaming should not be undertaken lightly. This feature is intended for special use cases, such as where accessions are created in a nursery with a name that is different from the accession name in the downstream breeding program. + +It can also be used to rename accessions in bulk that have spelling mistakes and other issues. Please note however, that the tool does not make any attempt to change the names of associated elements, such a plots, that may have been constructed using accession names. + +Because of the many implications of accession renaming, the feature is limited to accounts with the curator role. + +\hypertarget{managing-seed-lots}{% +\chapter{Managing Seed Lots}\label{managing-seed-lots}} + +Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. The seed in seedlots can be from crosses or can be named accessions. Seedlots from crosses would represent seed harvested. Click Manage and then Seed Lots to begin. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlots} \end{center} + +\hypertarget{add-new-seedlots}{% +\section{Add New Seedlot(s)}\label{add-new-seedlots}} + +To add a single new seedlot, click on ``Add Seedlot''. This will bring up the following dialog where you enter information about where the seedlot exists, what accession or cross is contained in it, and how many seeds there are. A seedlot must contain either an accession or a cross, and not both. A seedlot must have a weight in grams or a seed count or both of these. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_add_seedlot} \end{center} + +In the case where you have many seedlots to add to the database, you can upload an excel XLS or XLSX file instead. Click ``Upload Seedlots'' to see the following dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_upload_seedlots} \end{center} + +\hypertarget{seedlot-transactions}{% +\section{Seedlot Transactions}\label{seedlot-transactions}} + +Seedlots are capable of tracking where seeds came from, such as from crosses, and to where seeds go, such as to plots in the field. If you navigate to a seedlot detail page you will see the following. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_detail_page} \end{center} + +On this page you see and can edit information regarding a single seedlot, such as its name and location. You will also see a table indicating all t he transactions that a seedlot has been involved in, such as if it was planted in a plot in the field. Transactions to field plots are created when adding or uploading a new trial or from a trial's detail page. Clicking on ``Add New Transaction'' let you add a transaction from between this seedlot and another seedlot. This kind of transaction is useful for representing if you have distributed seed to different locations. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_add_new_transaction} \end{center} + +\hypertarget{seed-inventory}{% +\section{Seed Inventory}\label{seed-inventory}} + +To inventory your seed: 1) Make sure your seedlots are in the database. Use ``Add New Seedlot'' to add a single seedlot or ``Upload New Seedlots'' to add many. 2) Make sure your seedlots are barcoded. You can print these barcodes from the database. 3) Use the ``Inventory'' Android Application to scan seedlot barcodes and record weight. Then use ``Upload Inventory'' to upload this info into database. If you prefer you can create your own CSV file and upload that, if you do not want to use the Inventory Application. For more info about the ``Inventory'' Android Application go to Inventory. + +Clicking the ``Upload Inventory'' button will bring the following dialog: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_upload_inventory} \end{center} + +The CSV file that should contain your inventory should meet these Template requirements. The Seed Inventory Android Application exports this exact file. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_upload_inventory_template} \end{center} + +\hypertarget{find-seedlots-for-a-list-of-accessions}{% +\section{Find Seedlots For a List of Accessions}\label{find-seedlots-for-a-list-of-accessions}} + +A convenient tool for searching available seedlots for a list of accessions is available in the list tool. First open up your list of accessions. For help opening a list of accessions please see the List section help. There is a button called ``See Available Seedlots''. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/manage_seedlot_accession_list_search} \end{center} + +Once you click this, you will see the following table in a dialog. From here you can create a list of seedlots using the checkboxes and the input at the bottom. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_list_available_seedlots} \end{center} + +\hypertarget{create-a-seedlot-for-an-accession-or-cross}{% +\section{Create a seedlot for an Accession or Cross}\label{create-a-seedlot-for-an-accession-or-cross}} + +Complementary to what we saw above for creating seedlots from the ``Manage Seedlots'' page, it is possible to create a new seedlot from an accession's detail page or from the cross detail page. On the accession detail page, this is visible in the ``Related Stocks'' section as seen below. The cross detail page has an identical section. Notice the link for creating a new seedlot, which streamlines adding the seedlot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlot_accession_detail} \end{center} + +\hypertarget{add-quality-data-to-a-seedlot}{% +\section{Add quality data to a seedlot}\label{add-quality-data-to-a-seedlot}} + +Quality information can be added to a seedlot in the quality field. This is also available as a column in the file upload format. It is recommended to use a controlled vocabulary, defined by the user, for the quality field. For example, good quality seed should be labelled ``ok'', whereas other quality descriptors could be ``moldy'', ``insect damage'', or ``low sprouting'', etc. + +\hypertarget{seedlot-maintenance-events}{% +\section{Seedlot Maintenance Events}\label{seedlot-maintenance-events}} + +For some crops, such as sugar kelp, a ``seedlot'' requires routine maintenance for the successful long-term storage of the seedlot. (For example, a Seedlot Maintenance Event for sugar kelp would be the routine change of the water that gametophytes are kept it). Breedbase can now store a record of these Seedlot Maintenance Events associated directly with existing Seedlots. Maintenance Events can be uploaded using a simple Excel template or recorded directly on the website. + +\hypertarget{setup}{% +\subsection{Setup}\label{setup}} + +Each Breedbase instance needs to be configured to support the storage of Seedlot Maintenance Events since each crop will have their own distinct set of maintenance events for their seedlots. To check if your Breedbase instance supports this feature, go to the Manage menu and select the Seed Lots page. Make sure you are logged in and look for the \textbf{Seedlot Maintenance} button near the top, next to the \textbf{Create Seedlot(s)} and \textbf{Upload Inventory} buttons. If you don't see this button, contact the developer(s) supporting your Breedbase instance and ask if they can setup this feature. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_seedlots_seedlot_maintenance} \end{center} + +\emph{The location of the Seedlot Maintenance button on the Manage \textgreater{} Seed Lots page} + +\hypertarget{adding-events}{% +\subsection{Adding Events}\label{adding-events}} + +Seedlot Maintenance Events can be added using two methods: 1) Uploading an Excel template or 2) Recording events directly on the website + +\hypertarget{uploading-events-with-excel-template}{% +\subsubsection*{Uploading Events with Excel Template}\label{uploading-events-with-excel-template}} + + +To bulk-upload a file of Seedlot Maintenance Events, first create an Excel (.xls or .xlsx) file with the following headers: + +\begin{itemize} +\tightlist +\item + \textbf{seedlot} - the name of the Seedlot to associate the event with (must exactly match an existing Seedlot in the database) +\item + \textbf{type} - the name of the Seedlot Maintenance Event type (these vary between Breedbase instances, a list of supported event types is displayed on the upload page) +\item + \textbf{value} - the value of the Seedlot Maintenance Event (these may be different for each event type and vary between Breedbase instances, a list of supported event values is displayed on the upload page) +\item + \textbf{notes} - optional, additional notes/comments about the event +\item + \textbf{operator} - the username of the Breedbase user that recorded the event +\item + \textbf{timestamp} - the date/time the event was recorded, in `YYYY-MM-DD HH:MM:SS' format +\end{itemize} + +Once you have an Excel file with the events filled out, follow these steps to upload the events to the database: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Make sure you are logged in to your Breedbase instance +\item + Go to the Manage \textgreater{} Seed Lots page +\item + Select the \textbf{Seedlot Maintenance} button +\item + Select the \textbf{Upload Maintenance} button +\item + Choose your Excel (.xls or .xlsx) file to upload +\item + Select the \textbf{Upload} button +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_maintenance_upload} \end{center} + +\emph{The Seedlot Maintenance upload dialog, showing the supported event types and values (for sugar kelp)} + +\hypertarget{recording-events-on-website}{% +\subsubsection*{Recording Events on Website}\label{recording-events-on-website}} + + +To add individual Seedlot Maintenance Events to the database in real time, as they're being recorded, use the \textbf{Record Maintenance} page. Follow these steps to record Seedlot Maintenance Events: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Make sure you are logged in to your Breedbase instance +\item + Go to the Manage \textgreater{} Seed Lots page +\item + Select the \textbf{Seedlot Maintenance} button +\item + Select the \textbf{Record Maintenance} button +\item + Enter the \textbf{Seedlot Name} or scan a barcode that has the Seedlot Name encoded. Once entered, the box at the top of the page will display basic information about the Seedlot as well its recently recorded events. +\item + Select or Enter the values of individual events +\item + Optionally, notes button next to each event to add additional notes/comments about that specific event +\item + Make sure the operator/username and timestamp are correct +\item + Select the \textbf{Submit} button to add the recorded events to the database. NOTE: any events that remain selected as ``Not Recorded'' will not be submitted to the database. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_maintenance_record} \end{center} + +\emph{The Seedlot Maintenance record page, as configured for sugar kelp} + +\hypertarget{displaying-events}{% +\subsection{Displaying Events}\label{displaying-events}} + +Recently recorded Seedlot Maintenance Events are displayed in a table from the main Seedlot Maintenance page, as well as the detail page for individual Seedlots. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_maintenance_events_unfiltered} \end{center} + +\emph{Unfiltered table of recent Seedlot Maintenance events} + +The events displayed in these tables are sorted by timestamp, with the most recently recorded events displayed first. The displayed events can be filtered using any number of supported filter criteria, such as: - seedlot names (as entered on the page or using an existing seedlot list), - dates (on, on or before, before, on or after, and/or after the entered dates) - event types - event type values - operator/username + +Select the properties of the filter(s) you want to apply, then select the \textbf{Add} button next to the button to add the filter to the list of applied filters. Once you're done adding filters, select the \textbf{Filter} button to search the database for the filtered events. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/seedlot_maintenance_events_filtered} \end{center} + +\emph{A filtered table of Seedlot Maintenance events} + +The filtered events can be downloaded directly from the table using the \textbf{Excel} or \textbf{CSV} buttons at the top of the table. Or Seedlot Maintenance Events can be bulk-downloaded (this includes all events for a Seedlot) using a list of Seedlots from the main downloads page (see below). + +\hypertarget{downloading-events}{% +\subsection{Downloading Events}\label{downloading-events}} + +To bulk-download all events for a specific subset of Seedlots: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Create a list containing the Seelots you are interested in. +\item + Go to the \textbf{Download Using Lists} page (Manage \textgreater{} Download) +\item + Find the \textbf{Download Seedlot Maintenance Events} section +\item + Select your list of Seedlots +\item + Select the \textbf{Download} button to generate the download file +\end{enumerate} + +The downloaded file will follow the same format as the upload template and will contain all recorded Seedlot Maintenance Events for each Seedlot in the list. + +\hypertarget{deleting-seedlots}{% +\section{Deleting Seedlots}\label{deleting-seedlots}} + +Seedlots can be deleted on the Manage Seedlots page (/breeders/seedlots) by search the seedlot and then clicking the X to delete one seedlot at a time. To delete a seedlot, the logged in user needs the required delete privileges on the seedlot. The seedlot also should not have any transactions associated with it (except for the initial transaction). + +To delete seedlots in bulk, generate a list of type seedlot, for example, using the wizard. Open the section ``Delete seedlots using a list'' on the Manage Seedlots page and select the list. Seedlot deletion using a list is only available to user with curator status. + +\hypertarget{managing-populations}{% +\chapter{Managing Populations}\label{managing-populations}} + +Populations are modeled as groups of accessions. This grouping can be useful in downstream analyses. To manage these populations go to Manage Accessions and scroll tp the bottom. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_populations} \end{center} + +To add a new population click ``Create Population''. The following dialog will appear where you choose a list of accessions and give a name to the new population. Please note it is also possible to create a population when you are uploading new accessions into the database. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/manage_populations_add_population} \end{center} + +Click on the plus (+) button next to Populations to see all the available populations. Click on a population name to see the accessions in the population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_populations_expand_table} \end{center} + +From here you can delete accessions from a population as well as add new accessions to the population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_populations_table} \end{center} + +\hypertarget{managing-crosses}{% +\chapter{Managing Crosses}\label{managing-crosses}} + +Information for crosses can be managed using the ``Crosses'' option in the Manage menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image148} \end{center} + +\hypertarget{crossing-experiment}{% +\section{Crossing Experiment}\label{crossing-experiment}} + +Different crosses in the same trial/nursery/project are grouped via ``\textbf{crossing experiment}''. Crossing experiments are organized based on their breeding programs. To find a crossing experiment, you can either type the crossing experiment name in the ``Search'' box, or look for the crossing experiment directly in its breeding program by clicking on the ``\textbf{+}'' icon. In each breeding program, crossing experiments can be placed directly in the breeding program, or organized in folders. The ``\textbf{Folders}'' section allows you to place crossing experiments in folders, move a crossing experiment in a folder to another folder, or rearrange your folders within a breeding program. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_crosses1} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_crosses2} \end{center} + +\hypertarget{add-new-crossing-experiment}{% +\subsection{Add New Crossing Experiment}\label{add-new-crossing-experiment}} + +To add a new crossing experiment, click on ``Add Crossing Experiment'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_crosses_crossingtrial} \end{center} + +Required Information: + +• ``\textbf{Crossing Experiment Name}'': enter a name for the crossing experiment. The crossing experiment name must not already exist in the database. + +• ``\textbf{Breeding program}'': select a breeding program that is available in the database. New breeding programs can be added on the ``Breeding program'' page, accessible from the ``Manage'' menu. \emph{Breeding Program Page} + +• ``\textbf{Location}'': select a location for the crossing experiment. New locations can be entered on the ``\textbf{Locations}'' page, accessible from the ``\textbf{Manage}'' menu. \emph{Location Page} + +• ``\textbf{Year''}: select a year. + +• ``\textbf{Description}'': enter a description for the crossing experiment. + +After filling in the information, click ``\textbf{Submit}'' to generate the crossing experiment. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/add_crossingtrial} \end{center} + +\hypertarget{cross}{% +\section{Cross}\label{cross}} + +\hypertarget{add-new-crosses}{% +\subsection{Add New Crosses}\label{add-new-crosses}} + +\hypertarget{add-a-cross-by-using-the-add-new-cross-dialog}{% +\subsubsection*{Add a cross by using the ``Add New Cross'' dialog}\label{add-a-cross-by-using-the-add-new-cross-dialog}} + + +To add a single new cross, click on ``Add Cross'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_crosses_addcross} \end{center} + +Enter cross information in the popup dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/add_cross} \end{center} + +Required Information: + +• ``\textbf{Crossing experiment}'': select a crossing experiment available in the database. + +• ``\textbf{Location}'': select a location available in the database. + +• ``\textbf{Cross name}'': enter a name for the cross. The cross name must not already exist in the database. + +• ``\textbf{Cross type}'': the options for cross types are: biparental, self, open pollinated, bulk, bulk selfed, bulk and open pollinated, double haploid, polycross, reciprocal and multicross. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/new_cross_dialog_type} \end{center} + +• The ``\textbf{Female Parent''} and ``\textbf{Male Parent''} field are auto-complete fields for accessions that are already in the database. The parents specified will be entered in the pedigree of the new accessions generated by this cross. + +Optional Information: + +• ``\textbf{Female Plot and/or Male Plot}'': In addition to the accession names, specific plots used in the cross can also be added to the database. To retrieve plot names associated with each female/male accession, enter your trial name, then click ``\textbf{Search Plots}''. Plot names of each parental accession in that field trial will be shown in the drop-down list, you can then select the plot used in the cross. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/new_cross_dialog2} \end{center} + +Additional crossing experimental information such as pollination date, number of flowers, number of fruits, number of seeds can be specified during adding new cross. Alternatively, this information can be updated or edited directly on the ``\textbf{Cross Details}'' page. + +If you know the number of accessions that are generated from the cross, they can be instantiated immediately in the database by clicking the ``\textbf{Add accessions for progeny}'' checkbox and specifying the number. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/add_cross_info} \end{center} + +Click ``Submit'' to generate the cross. + +\hypertarget{upload-new-crosses}{% +\subsubsection*{Upload New Crosses}\label{upload-new-crosses}} + + +To upload new crosses from an Excel file (.xls or .xlsx), click on ``Upload Crosses'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_crosses_upload} \end{center} + +Select a crossing experiment and a location available in the database from drop-down lists and choose a file that you want to upload, then click ``\textbf{Upload File}''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/upload_crosses} \end{center} + +Please check spreadsheet format carefully. The file must be an Excel file (.xls or .xlsx). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_upload_format} \end{center} + +\hypertarget{update-crosses-by-uploading}{% +\subsection{Update Crosses by Uploading}\label{update-crosses-by-uploading}} + +To upload progenies and/or experimental info of crosses already in the database, go to ``\textbf{Manage-Upload}'' page. + +In the ``\textbf{Crosses}'' section, there are links for uploading progenies and experimental info. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_upload_page} \end{center} + +Please check spreadsheet format in each link carefully. The file must be an Excel file (.xls or .xlsx). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/progenies_upload_spreadsheet} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/crossinfo_upload_spreadsheet} \end{center} + +Note: crossing experimental information is customized based on the need for each crop. As a result, column headers for experimental info in your database may be different from the information shown in this manual. + +\hypertarget{cross-wishlist}{% +\section{Cross Wishlist}\label{cross-wishlist}} + +An Android ODK application is being developed to record cross information on a mobile device in the field. To link this mobile application with the database, the Cross Wishlist can be used to create a plan for which crosses to perform. + +This tool is available on the Manage Cross page. It is currently only available on certain databases, so when you click this link you may see an alert mentioning that the cross wishlist is not available on your database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_cross_click_cross_wishlist} \end{center} + +\hypertarget{create-a-cross-wishlist}{% +\subsection{Create a Cross Wishlist}\label{create-a-cross-wishlist}} + +\hypertarget{step-1.-select-the-accessions-to-be-crossed-in-your-trial}{% +\subsubsection*{Step 1. Select the accessions to be crossed in your trial}\label{step-1.-select-the-accessions-to-be-crossed-in-your-trial}} + + +There are two interfaces for this step, either ``Not Using Lists'' or ``Using Lists''. Depending on if you already have a list of female and male accessions to use, you can decide on which interface to use. The end result of using either interface is the same. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_initial_dialog} \end{center} + +We will start by showing ``Not Using Lists''. First select the trial in which the crosses are to be performed. This will populate a select box with all the accessions used in that trial. From here, one or many accessions can be selected as the female accession. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_not_using_list_01} \end{center} + +Once the female accessions are selected, a table is populated. Each row in this table begins with the female accession that was selected, followed by a select box with all the accessions used in the trial. From here, one or many accessions can be selected as the male to use in the cross. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_not_using_list_02} \end{center} + +Once the male accessions are selected to cross with each female accession, a table indicating priorities appears. Priority is meant to indicate an order in which to attempt the cross; first the highest priority male will be considered, but if this cross is not possible then subsequent males will be considered. An equal priority can be given and this will not indicate a specific order to follow. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_not_using_list_03} \end{center} + +Alternatively, we could have used the ``Using List'' interface instead. Here we select the trial in which the crosses will be performed and we provide a list of accessions to consider for the females and the males to be crossed. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_using_list_01} \end{center} + +\hypertarget{step-2.-select-the-female-plots-to-be-considered-in-the-crosses}{% +\subsubsection*{Step 2. Select the female plots to be considered in the crosses}\label{step-2.-select-the-female-plots-to-be-considered-in-the-crosses}} + + +After selecting your lists, the table below is populated. The first column has all the female accessions specified and the header row has all the male accessions specified. The males to consider crossing with each female are indicated with priority. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_using_list_02} \end{center} + +After female and male accessions are selected to cross, either by the ``Nor Using List'' or ``Using List'' interface, click Next. The next dialog will allow selection of specific female plots to use for the cross. Sections for each female accession selected will appear with the field layout displayed. Selecting all plots in which the female is present indicates that the cross should be performed on all plots where that female accession is present. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_wishlist_not_using_list_04} \end{center} + +\hypertarget{step-3.-transfer-the-cross-wishlist-to-your-mobile-crossing-application}{% +\subsubsection*{Step 3. Transfer the cross wishlist to your mobile crossing application}\label{step-3.-transfer-the-cross-wishlist-to-your-mobile-crossing-application}} + + +Clicking ``Push Cross Wishlst for ODK Use'' will send the cross wishlist plan to the ONA server for use by the mobile ODK application. Crosses can then be performed and recorded in the field using the mobile application. Afterwards, the crosses are sent back to our database and stored. + +\hypertarget{crossing-experiment-detail-page}{% +\section{Crossing Experiment Detail Page}\label{crossing-experiment-detail-page}} + +Information for crosses in the same crossing experiment is compiled in the crossing experiment detail page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/crossingtrial_1} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/crossingtrial_2} \end{center} + +Each cross name, female parent, male parent, female plot and male plot has a link to its own detail page, which contains information specific to each one. Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. + +\hypertarget{cross-detail-page}{% +\section{Cross Detail Page}\label{cross-detail-page}} + +Information of each cross can also be viewed in its detail page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_page_1} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_page_2} \end{center} + +This page allows you to update or edit crossing experimental information and add progenies related to that cross. Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/edit_cross_info} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cross_add_progenies} \end{center} + +\hypertarget{managing-field-trials}{% +\chapter{Managing Field Trials}\label{managing-field-trials}} + +To view trial details on the database, click on the ``Field Trials'' link under the ``manage'' menu on the toolbar. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image290} \end{center} + +Clicking on the ``Field Trials'' link will bring you to the ``Manage Trials'' page. On this page, trials are organized according to their breeding programs. To access trial details, click on the + icon next to your breeding program. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image153} \end{center} + +Trials can be placed directly in their breeding program. Alternatively, they can be organized by using folders within each breeding program. Clicking on trial name will take you directly to the trial details page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image279} \end{center} + +\hypertarget{trial-detail-page}{% +\section{Trial Detail Page}\label{trial-detail-page}} + +The trial detail page displays important information about individual trials including breeding program, location, year, description of the trial, design, and any files associated with that trial. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_start} \end{center} + +Below the trial details you will find various menus for accessing and modifying trial data. There are sections for printing labels for your plots or plants, recording phenotypes, viewing your trial layout or design, viewing phenotypes for this trial, or conducting analyses. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_navigator_1} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_navigator_2} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_navigator_3} \end{center} + +The ``transplanting date'' field feature will only be displayed if it has a value. To add a transplanting date after creating a trial, change the show\_transplanting\_date parameter from 0 to 1 in the SGN config file. As a result, you will be able to add a date under the transplanting date field by clicking the ``Edit Trial Details'' on the trial detail page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/add_transplanting_date} \end{center} + +\hypertarget{adding-trials}{% +\section{Adding Trials}\label{adding-trials}} + +Only users with the account status of ``submitter'' may create trials. To learn how to change your account status from ``user'' to ``submitter'' visit section \ref{managing-your-account}. + +\hypertarget{prerequisites}{% +\subsection{Prerequisites}\label{prerequisites}} + +\begin{itemize} +\item + To add a trial, all of your accessions should already exist in the database before you begin to design a trial. If you have accessions that are not in the database, see the instructions in \protect\hyperlink{managing-accessions}{\emph{Managing Accessions}}. +\item + The breeding program and location for your trial should also exist in the database. If you need to add breeding program and/or location to the database, see the instructions in \protect\hyperlink{managing-breeding-programs}{\emph{Managing Breeding Programs}} and \protect\hyperlink{managing-locations}{\emph{Managing Locations}} respectively. +\end{itemize} + +On the ``Manage Trials'' page, there are two methods to create trials: by selecting ``Upload Existing Trial(s)'' to create a trial or trials from a spreadsheet; or by selecting ``Design New Trial'' and entering the trial data by hand. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_manage_trials} \end{center} + +\hypertarget{adding-a-trial-by-using-design-new-trial-form}{% +\subsection{Adding a trial by using ``Design New Trial'' form}\label{adding-a-trial-by-using-design-new-trial-form}} + +\hypertarget{step-1.-begin-the-design-new-trial-workflow}{% +\subsubsection*{Step 1. Begin the ``Design New Trial'' workflow}\label{step-1.-begin-the-design-new-trial-workflow}} + + +Click on ``Design New Trial'' to begin. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_open_form} \end{center} + +The first step in this workflow is an introduction that looks like this: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_1} \end{center} + +Here it gives information about what is required for a trial, including that to create a new trial, you need to create a list of the accessions that you would like to use in the trial. Lists can be viewed, created, and modified with the ``lists'' tool at the upper right of the screen. For more information on lists, click \protect\hyperlink{working-with-lists}{here}. + +\hypertarget{step-2.-enter-trial-information}{% +\subsubsection*{Step 2. Enter Trial Information}\label{step-2.-enter-trial-information}} + + +On this screen you need to enter basic information about the trial, such as breeding program and location(s). You must also select a design type, such as Complete Block Design. The design is important because it influences how your genotypes are distributed and randomized over the trial. You must first click validate before proceeding to the next step. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_2} \end{center} + +\hypertarget{step-3.-enter-design-information}{% +\subsubsection*{Step 3. Enter Design Information}\label{step-3.-enter-design-information}} + + +On this screen you need to specify a list of accessions to use in the experiment. This list must be a valid list of accessions. You must also specify all required design information, such as number of replicates. In this case, the number of blocks must be given. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_3} \end{center} + +\hypertarget{step-4.-enter-trial-linkage-information-optional}{% +\subsubsection*{Step 4. Enter Trial Linkage Information (Optional)}\label{step-4.-enter-trial-linkage-information-optional}} + + +This next sections allows you to associate this new trial with other field trials, crossing experiments, or genotyping plates already present in the database. This is optional, and can be completed at a later date from the trial detail page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_4} \end{center} + +\hypertarget{step-5.-enter-field-map-information-optional}{% +\subsubsection*{Step 5. Enter Field Map Information (Optional)}\label{step-5.-enter-field-map-information-optional}} + + +On this screen you can specify how the row and column numbers will be generated for the plots in the trial. The row and column number represent a relative position of the plot in the field. If you are not exactly sure of how you will plant the plots in the field or you have an irregular (non-rectangular) layout, you can skip this step for now. This information can be added on the Trial Detail Page once the trial is saved in the database in order to reflect exactly how the plots were planted in the field. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_5} \end{center} + +\hypertarget{step-6.-custom-plot-naming-optional}{% +\subsubsection*{Step 6. Custom Plot Naming (Optional)}\label{step-6.-custom-plot-naming-optional}} + + +On this screen it is possible to change the format in which plot names will be generated for your trial. It is recommended to skip this step and just use the format generated by the database by default. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_6} \end{center} + +\hypertarget{step-7.-review-designed-trial}{% +\subsubsection*{Step 7. Review Designed Trial}\label{step-7.-review-designed-trial}} + + +On this screen you can review the trial that the database has generated. + +You will see a graphical representation of the trial. The numbers on the squares represent the plot\_number of each plot and on mouse hover you can see further information about the plot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_7_top} \end{center} + +You will also see a table representation of all the plots and their information. If you want to redo the randomization, you can click the ``Redo Randomization'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_7_middle} \end{center} + +At the bottom there is a brief summary of the trial followed by two buttons. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_7_bottom} \end{center} + +\hypertarget{step-8.-add-field-management-factors-to-your-design-optional}{% +\subsubsection*{Step 8. Add Field Management Factors to your design (Optional)}\label{step-8.-add-field-management-factors-to-your-design-optional}} + + +You can add Field Management Factors by clicking ``Add Field Management Factor(s) to Design''. Clicking this opens a dialog to specify your factor. You can name this to account for fertilizer, watering regime, inoculation, or anything else. The types of management factors available is customizable in the SGN configuration file. This is optional and can be added from the trial detail page afterwards. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/add_management_factor_name_dialog} \end{center} + +Click ``Continue'' and a dialog will appear where you can specify plots or plants (if you added plants during trial creation) for which the factor was applied. There is a select all button as well. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/add_management_factor_dialog} \end{center} + +\hypertarget{step-9.-saving-new-trial-in-the-database}{% +\subsubsection*{Step 9. Saving new trial in the database}\label{step-9.-saving-new-trial-in-the-database}} + + +Once you are done reviewing the trial you can click ``Confirm'' to save the generated trial into the database. Once the trial has saved you will see the final completion screen: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_form_8} \end{center} + +\hypertarget{adding-a-trial-from-an-uploaded-file}{% +\subsection{Adding a trial from an uploaded file}\label{adding-a-trial-from-an-uploaded-file}} + +If you already have trial design layout in a spreadsheet, you can add your trial into the database by using the ``Upload Existing Trial(s)'' button on the \emph{Manage Trials} page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_create_manage_trials} \end{center} + +Enter the information in the workflow to upload a trial from a spreadsheet. + +\hypertarget{step-1}{% +\subsubsection*{Step 1:}\label{step-1}} + + +The first step is to understand what the format of the trial upload is. It is important to understand that the field layout represents plots in the experiment. Each plot has a globally unique plot\_name, a sequential plot\_number that is unique in the trial (but not globally unique. e.g.~101, 102, 103 for three separate plots), an accession\_name representing what genotype is planted in that plot, and a block\_number representing design replication. Each plot can be thought of as having a row\_number and a column\_number representing the relative position of the plot in a grid (e.g.~the top left plot is row 1 column 1 following by row 1 column 2). Each plot can be planted with an amount of seed from a seedlot, where the seedlot\_name represents the specific seed packet that was used, and num\_seed\_per\_plot and weight\_gram\_seed\_per\_plot represent amount that were transferred from the seedlot\_name to the plot\_name. Treatments can be applied onto plots using additional column names in your file, where a 1 represents if the treatment was applied to the plot and an empty cell means it was not applied. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_1} \end{center} + +The following page will allow you to pick a file for upload, including uploading multiple trials at once. On this page you can also inspect the file requirements for both single trial and multi-trial uploads. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_2} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_template} \end{center} + +\hypertarget{minimum-file-requirements}{% +\paragraph*{Minimum File requirements}\label{minimum-file-requirements}} +\addcontentsline{toc}{paragraph}{Minimum File requirements} + +\begin{itemize} +\item + All accession names in the file must exist in the database. See adding accessions for more information. +\item + The uploaded file can be excel (.XLXS or .XLS), comma-separated values (.CSV), tab-separated values (.TSV) or semicolon-separated values (.SSV). +\item + The first row (header) must contain the column names: plot\_name accession\_name plot\_number block\_number is\_a\_control rep\_number range\_number row\_number col\_number seedlot\_name num\_seed\_per\_plot weight\_gram\_seed\_per\_plot entry\_number +\item + Only accession\_name, plot\_number, and block\_number are required. +\end{itemize} + +Minimal Example: + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0611}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0802}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0687}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0725}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0763}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0649}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0725}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0649}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0649}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0725}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0992}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.1336}} + >{\raggedright\arraybackslash}p{(\columnwidth - 24\tabcolsep) * \real{0.0687}}@{}} +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +\textbf{plot\_name} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{accession\_name} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{plot\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{block\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{is\_a\_control} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{rep\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{range\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{row\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{col\_number} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{seedlot\_name} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{num\_seed\_per\_plot} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{weight\_gram\_seed\_per\_plot} +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +\textbf{entry\_number} +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +2018plot1 & my\_accession1 & 101 & 1 & 1 & & & & & & & & \\ +2018plot2 & my\_accession2 & 201 & 2 & & & & & & & & & \\ +2018plot3 & my\_accession2 & 102 & 1 & & & & & & & & & \\ +2018plot4 & my\_accession1 & 202 & 2 & 1 & & & & & & & & \\ +\end{longtable} + +When uploading multiple trials, the requirements are the same, but with additional headers to differentiate trials in the same file: trial\_name breeding\_program location year transplanting\_date design\_type description trial\_type trial\_stock\_type plot\_width plot\_length field\_size planting\_date harvest\_date. Additionally, when uploading multiple trials, you may choose to get an email when the upload is complete (see section \ref{email-alert-for-multiple-trial-design-upload} ) + +\hypertarget{file-validation}{% +\paragraph*{File validation}\label{file-validation}} +\addcontentsline{toc}{paragraph}{File validation} + +\begin{itemize} +\tightlist +\item + In case of errors in the uploaded file such as missing or invalid data, a window will appear listing the specific errors in the file that must be corrected before a successful upload. +\end{itemize} + +\hypertarget{uploading-a-trial-with-treatments}{% +\paragraph*{Uploading a trial with Treatments}\label{uploading-a-trial-with-treatments}} +\addcontentsline{toc}{paragraph}{Uploading a trial with Treatments} + +\begin{itemize} +\tightlist +\item + You can upload a trial with treatments by adding additional column(s). The column header will be the treatment e.g.~fertilizer, watering regime, inoculation, etc. and the values in these columns will be either 1 or empty, indicating that the treatment was applied to the plot or not. +\end{itemize} + +\hypertarget{step-2}{% +\subsubsection*{Step 2:}\label{step-2}} + + +Once you feel that your experiment field layout is in the right format, click on to the Next Step. You will see the following form which must be filled in completely: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_3} \end{center} + +The trial name must be globally unique in the database. Please try to follow standard naming conventions for your group. + +\hypertarget{step-3}{% +\subsubsection*{Step 3:}\label{step-3}} + + +Go to the next page where you can link this trial to other projects. Validate the form, and then you can click ``Upload Trial''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_4} \end{center} + +\hypertarget{step-4}{% +\subsubsection*{Step 4:}\label{step-4}} + + +In the case where you have uploaded an experiment using accession\_names that are not already present in the database, you will be taken to this screen. If the accession\_names in your file are all already in the database, this step will be skipped. The reason it is necessary for your accessions to be in the database before you can add a trial using them is that a single accession can be used among many trials and therefore must exist as a separate entity in the database; because of this it is also very important to be careful about adding wrongly duplicated accession\_names into the database. From this screen it is possible to make a new list with the missing accession\_names and then click ``Add Accessions to the database'' to immediately resolve the issue. Once all your accessions are in the database, click to move to the Next Step. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_5} \end{center} + +\hypertarget{step-5}{% +\subsubsection*{Step 5:}\label{step-5}} + + +In the case where you have uploaded an experiment using seedlot\_names that are not already present in the database, you will be taken to the next screen. If the seedlots in your file are all already in the database, this step will be skipped. The reason it is necessary for your seedlots to be in the database before you can add a trial using them is that a single seedlot can be used among many trials and therefore must exist as a separate entity in the database. Once all your seedlots are in the database, click to move to the next step. + +\hypertarget{step-6}{% +\subsubsection*{Step 6:}\label{step-6}} + + +If there are any other errors with your file, such as if the plot\_names are not globally unique in the database or your plot\_numbers are not unique in your trial or row\_number is not an integer or any other error, you will see the errors listed in the red box. It is up to you to correct these errors in your file. Simply open up the file you selected earlier in Excel and correct the issues and then save the file. Then you can click ``Submit Trial'' and it will resubmit it for you. You can continue to edit your file here and submit as many times as you need until it is accepted. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_7} \end{center} + +\hypertarget{completion-screen}{% +\subsubsection*{Completion screen}\label{completion-screen}} + + +Whether you were lucky enough to submit your trial successfully on Step 2 or if you tried many times on Step 5, once your trial has been saved in the database you will see the following screen: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_upload_trial_8} \end{center} + +\hypertarget{multi-location-trials}{% +\subsection{Multi-location trials}\label{multi-location-trials}} + +To add multi-location trials, simply select the multiple locations while using the `Add Trial' form. + +This will create a separate trial for each selected location, but they will share the same design and will be grouped in a single folder. + +By default each trial design will have a fresh randomization, but if desired you may check the ``Use same randomization for all locations'' option. + +\hypertarget{email-alert-for-multiple-trial-design-upload}{% +\subsection*{Email alert for multiple trial design upload}\label{email-alert-for-multiple-trial-design-upload}} + + +When uploading multiple trials from a file, you have the option to receive email notifications by clicking the ``Email Alert'' checkbox. By default, the system will use the email address associated with your account, but you have the option to enter a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/multiple_trial_upload_with_email} \end{center} + +\hypertarget{viewing-plot-layout-and-trait-heatmap}{% +\subsection{Viewing Plot Layout and Trait HeatMap}\label{viewing-plot-layout-and-trait-heatmap}} + +\hypertarget{viewing-plot-layout}{% +\subsubsection{Viewing plot layout}\label{viewing-plot-layout}} + +In the ``Field Layout Tools and Phenotype Heatmap'' section of a Trial Detail page, the trial physical layout is displayed by default. The relative position of the plots will be displayed based on the row and column positions given to the plots during the trial creation or upload steps. The plots are color-coded based on the plot's rep and block numbers and whether or not it is used as a check. Hover the mouse over the plot to see details about a specific plot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_trial_layout} \end{center} + +\hypertarget{viewing-plot-layout-for-multiple-trials}{% +\subsubsection{Viewing plot layout for multiple trials}\label{viewing-plot-layout-for-multiple-trials}} + +If there is more than one trial grown in the same physical field, the trial layouts of all of the trials can be shown together if the trials share these properties: + +Each trial has the same year + +Each trial has the same location + +The location type of the trials' location is set to Field + +The row and column positions of all of the plots (across the related trials) don't overlap. For example, trial \#1 starts at row 1 and trial \#2 starts at row 10. + +When these conditions are met and you check the ``Select Trials in Same Field'' checkbox, the plots from all of the related trials will be displayed on the same field layout. The plots will be color-coded by trial. The planting order and harvest order downloads will include the plots from all of the displayed trials in the order in which the plots occur in the field. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_multi_trial_layout} \end{center} + +\hypertarget{tracking-plot-images-on-fieldmap}{% +\subsubsection*{Tracking plot images on fieldMap}\label{tracking-plot-images-on-fieldmap}} + + +Plot images can be seen on fieldMap if a plot is associated to any image. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_plot_image} \end{center} + +To view plot image(s), click on a plot, a dialog will appear. In this dialog you will see a detailed overview of the plot, including what stock(s) are contained within it and their layout in the plot, if applicable. At the bottom of the dialog, you will see options to change the plot name or accession as well as view plot images. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_view_plot_image} \end{center} + +On the resulting dialog, click on ``View Plot Images.'' To see more images if a plot has more that 2 images, click on See more images\ldots{} Medium size of an image can be viewed by clicking on an image. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_display_plot_image} \end{center} + +\hypertarget{viewing-assayed-trait-heatmap}{% +\subsubsection*{Viewing assayed trait heatmap}\label{viewing-assayed-trait-heatmap}} + + +The phenotype heatmap can be viewed by selecting a specific assayed trait from the selectbox drop-down. Mousing over the plots highlights the plot in green and also displays the plot's field information including the selected trait's phenotype value. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/heatmap_assayed_trait_view} \end{center} + +\hypertarget{suppressing-plot-phenotype}{% +\subsubsection*{Suppressing Plot Phenotype}\label{suppressing-plot-phenotype}} + + +Clicking on a plot on the heatmap would display a dialog that has a button for suppressing a plot phenotype value for a given trait. A suppressed plot value can be excluded during trial analysis and phenotype download. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_suppress_phenotype} \end{center} + +\hypertarget{correcting-spatial-autocorrelation}{% +\subsubsection{Correcting spatial autocorrelation}\label{correcting-spatial-autocorrelation}} + +For trials with spatial layout information and stored phenotypes, you can check and correct for spatial autocorrelation by clicking the ``Calculate Spatial Correction'' button above the field map. Doing so will open the spatial corrections dialog. For an in-depth review, visit the chapter on \protect\hyperlink{data-analysis-tools}{\emph{data analysis tools}}. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/heatmap_spatial_correction_button} \end{center} + +\hypertarget{adding-additional-information-in-the-trial-detail-page}{% +\subsection{\texorpdfstring{Adding additional information in the \emph{Trial Detail} page}{Adding additional information in the Trial Detail page}}\label{adding-additional-information-in-the-trial-detail-page}} + +After you added a new trial to the database, you can edit trial details or add more information for that trial through the \emph{Trial Detail} page. + +\#\#\#\# Uploading Physical Trial Layout \{-\} + +You can upload physical trial layout by clicking on the ``Upload Spatial Layout'' button on the \emph{Trial Detail} page. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/image332} \end{center} + +Please check file format carefully. You can find file format information by clicking on the ``Spreadsheet format'' on the ``Upload Spatial Layout'' window. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image340} \end{center} + +Spreadsheet format: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image190} \end{center} + +\hypertarget{physical-trial-layout-file-requirements}{% +\subsubsection*{Physical Trial Layout File requirements}\label{physical-trial-layout-file-requirements}} + + +\begin{itemize} +\item + All plot names in the file must exist in the database. +\item + The uploaded file should be tab delimited (txt). +\item + The first row (header) must contain the column names +\end{itemize} + +Example: + +\begin{longtable}[]{@{}lll@{}} +\toprule\noalign{} +plot\_name & row\_number & col\_number \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +plot1 & 1 & 1 \\ +plot2 & 1 & 2 \\ +plot3 & 1 & 3 \\ +\end{longtable} + +Select the trial layout coordinates file that you want to upload for this trial, then click ``OK'' button to upload the file. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image79} \end{center} + +The following message is displayed after the coordinates are uploaded. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image276} \end{center} + +\hypertarget{downloading-field-map-spreadsheet}{% +\subsubsection*{Downloading Field Map Spreadsheet}\label{downloading-field-map-spreadsheet}} + + +Field map spreadsheet can be downloaded if the trial has field coordinate (row and column numbers) uploaded for it plots. To download, click on the Download FieldMap Layout link on the Trial Heatmap section. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/field_map_download_link} \end{center} + +\hypertarget{editing-physical-trial-layout}{% +\subsubsection*{Editing Physical Trial Layout}\label{editing-physical-trial-layout}} + + +The ``Usage Help'' button contains information on how to edit physical trial layout. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/spatial_layout_usage_help} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/spatial_layout_usage_help_1} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/spatial_layout_usage_help_2} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/spatial_layout_usage_help_3} \end{center} + +There are two different options for editing trial layout: + +\begin{itemize} +\item + Replacing plot accession by clicking on the plot in the layout. +\item + Replacing trial accession by using the ``Edit Field Layout'' button. +\end{itemize} + +To edit a specific plot, click on that plot. Enter a new accession on the ``Edit Plot Info'' form, then click the ``Replace Plot Accession'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/fieldmap_view_plot_image} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/replace_plot_accession_form} \end{center} + +To replace an accession (in every plot/plant of that accession), click on the ``Edit Field Layout'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image173} \end{center} + +On the ``Edit Field Layout'' window, click the ``Replace Accession'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image87} \end{center} + +Select any accession that you want to replace, enter your new accession, then click the ``Replace Trial Accession'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image85} \end{center} + +\hypertarget{downloading-the-trial-layout-from-the-trial-detail-page}{% +\subsection{\texorpdfstring{Downloading the Trial Layout from the \emph{Trial Detail} page}{Downloading the Trial Layout from the Trial Detail page}}\label{downloading-the-trial-layout-from-the-trial-detail-page}} + +Click on ``Download Layout'' on the \emph{Trial Detail} page under ``Experimental Design''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image348} \end{center} + +The trial layout includes all information regarding the observation units in the experiment. The observation units can be plots, plants, or subplots. The trial layout can include trial design information such as the block\_number and rep\_number. It can also include physical map information such as the row\_number and col\_number, if that information is available for the trial. The trial layout also includes information regarding treatments that have been applied in the field. Optionally, the layout can give information regarding accession's global performance for a list of traits. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image347} \end{center} + +\hypertarget{adding-plant-entries-to-your-trial}{% +\subsection{Adding Plant Entries To Your Trial}\label{adding-plant-entries-to-your-trial}} + +After you added a new trial to the database you can choose to add plant entries to your trial. Adding plant entries enables plant level phenotyping. It is generally better to enter data at the plant level into the database because it is always possible to calculate plot level phenotypes from the individual plant data. + +Plant entries can be added to your trial in four ways: (1) Automatically generated by the database. The only input required is the number of plants per plot. (2) Uploaded in an XLS or XLSX file. This allows you to specifically name your plant entries. (3) Uploaded using plant index number instead of name (the name is automatically generated) (4) Uploaded using the number of plants per plot. This allows different numbers of plant per plot, as opposed to option 1. + +Additionally, if a trial has had subplots added, each of these options can be used to add plants to subplots. Note that once subplots are added, plants cannot be added directly to plots, and once plants have been added, subplots cannot be created. + +These options are available in the ``Plant Entries'' section on the \emph{Trial Detail} page under ``Experimental Design,'' as shown below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_add_plant_entries} \end{center} + +\hypertarget{automatically-generate-plant-entries}{% +\subsubsection*{Automatically Generate Plant Entries}\label{automatically-generate-plant-entries}} + + +Clicking on ``Add plant entries'' opens the following dialog box. The only input required is the number of plants per plot. This will create plant entries that are named as a concatenation of the plot\_name and the plant's index number e.g.~plot\_name\_plant\_1. You may optionally add row and column data to give plants a spatial layout within each plot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_add_plant_entries_autogenerated} \end{center} + +\hypertarget{upload-plant-entries}{% +\subsubsection*{Upload Plant Entries}\label{upload-plant-entries}} + + +Alternatively, you can choose to upload an XLS or XLSX file that contains the names of the plant entries, the plant index numbers, or the number of plants per plot. Each option comes with a dialog that specifies the file formats. + +\hypertarget{adding-tissue-sample-entries-to-your-trial}{% +\subsection{Adding Tissue Sample Entries To Your Trial}\label{adding-tissue-sample-entries-to-your-trial}} + +Some trials require tissue samples to be collected from plants in a field trial. The database will generate these tissue sample identifiers for you and will maintain all relationships with the plant, plot, accession, etc. To begin, go to the Design section of a trial's detail page and open the ``tissue sample entries'' section. Please note that tissue samples are directly related to plants, therefore your trial requires plants before you can add tissue samples. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_tissue_sample_default} \end{center} + +When you click on ``Add tissue sample entries'' you will see a dialog where you specify the number of tissue samples you require per plant. Once you have specified how many tissues samples, you can give specific words to distinguish samples, such as ``root'' or ``stem'', as seen below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_tissue_sample_create} \end{center} + +Once you have added tissue sample entries they will appear in the design section of the trial as seen below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_tissue_samples} \end{center} + +Each tissue sample has a detail page where you can add information about the sample, such as if it is in transit or in storage somewhere. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_tissue_sample_detail} \end{center} + +The related stocks section near the bottom of this detail page displays the relationships between all stocks, including tissue samples. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_trials_tissue_sample_related_stock} \end{center} + +\hypertarget{uploading-gps-coordinates-for-plots}{% +\subsection{Uploading GPS Coordinates For Plots}\label{uploading-gps-coordinates-for-plots}} + +You can upload GPS coordinates for the plots in your trial. There is a link on the Trial Detail Page as shown below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_add_plot_gps} \end{center} + +Clicking on this link will bring up the following dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_add_plot_gps_dialog} \end{center} + +Here you can upload an XLS or XLSX file. To see information on the format of the file that should be uploaded, click on ``Spreadsheet format''. This will bring up the following dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_add_plot_gps_upload_info} \end{center} + +This dialog tells you that the file must be XLS or XLSX and must contain: plot\_name WGS84\_bottom\_left\_x WGS84\_bottom\_left\_y WGS84\_bottom\_right\_x WGS84\_bottom\_right\_y WGS84\_top\_right\_x WGS84\_top\_right\_y WGS84\_top\_left\_x WGS84\_top\_left\_y The GPS coordinates should be WGS84 format and specify a four-pointed polygon around the plot. + +\hypertarget{repetitive-measurements-section}{% +\subsection{Repetitive Measurements Section}\label{repetitive-measurements-section}} + +If a trial includes repetitive traits or time-series values, you can effectively view and analyze these values through the Repetitive Measurements Section. Start by selecting the desired trait from the trait drop-down menu. Next, define the date range by either using the date-range picker or an interactive slider, which allows you to dynamically adjust the period you wish to examine. Once the date range is set, determine how to handle the repetitive measurements by choosing from various options such as First Value, Last Value, Averaged Value, Sum Values, or All Values. Choosing the ``All Values'' option enables an additional feature that visualizes the trend of the values over time, helping you identify patterns and trends within the data. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_view_repetitive_measurements} \end{center} + +\hypertarget{uploading-additional-files-to-trial}{% +\subsection{Uploading Additional Files To Trial}\label{uploading-additional-files-to-trial}} + +It may be of interest to you to upload additional documents, images, or recordings to your trial. To do this, scroll down to the ``Uploaded Additional File'' section on the trial detail page. From here you can view and download any of these additional files. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_upload_additional_file} \end{center} + +To upload an additional file, click on the ``Upload Additional Files'' link. A dialog will appear where you simply select your desired file. For information, you can click ``Upload information'' to see the following message. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_upload_additional_file_info} \end{center} + +\hypertarget{updating-trial-data}{% +\section{Updating Trial Data}\label{updating-trial-data}} + +To updated the trial-level metadata (such as the planting date, design type, description, etc) of one or more existing trials, click the ``Update Existing Trial(s)'' button from the Manage \textgreater{} Field Trials page. This upload can also be used to rename trials or move trials to a different breeding program. In order to update a trial, you must be a curator or a submitter (that is associated with the breeding program of the trials). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/update_trial_metadata} \end{center} + +Here you can upload a file that contains the new metadata for the existing trials in the database. The first column is labeled `trial\_name' and includes the name of the existing trial. Additional columns can be included for the metadata you want to update. Any columns not included in the file or values left blank will leave the existing metadata unchanged. The columns that can be included are: + +\begin{itemize} +\tightlist +\item + new\_trial\_name: A new name for the trial, must not already exist in the database +\item + breeding\_program: The name of breeding program that managed the trial, must exist in the database. +\item + location: The name or abbreviation of the location where the trial was held, must exist in the database. +\item + year: The year the trial was held. +\item + transplanting\_date: The transplanting\_date of the trial was conducted. Date in YYYY-MM-DD format or `remove' to remove the date +\item + planting\_date: Date of Planting in YYYY-MM-DD format or `remove' to remove the date +\item + harvest\_date: Date of Harvest in YYYY-MM-DD format or `remove' to remove the date +\item + design\_type: The shorthand for the design type, must exist in the database. Possible values include CRD: Completely Randomized Design, RCBD: Randomized Complete Block Design, RRC: Resolvable Row-Column, DRRC: Doubly-Resolvable Row-Column, ARC: Augmented Row-Column, Alpha: Alpha Lattice Design, Lattice: Lattice Design, Augmented: Augmented Design, MAD: Modified Augmented Design, greenhouse: undesigned Nursery/Greenhouse, splitplot: Split Plot, p-rep: Partially Replicated, Westcott: Westcott Design +\item + description: Additional text with any other relevant information about the trial. +\item + trial\_type: The name of the trial type, must exist in the database. Possible values include Seedling Nursery, phenotyping\_trial, Advanced Yield Trial, Preliminary Yield Trial, Uniform Yield Trial, Variety Release Trial, Clonal Evaluation, genetic\_gain\_trial, storage\_trial, heterosis\_trial, health\_status\_trial, grafting\_trial, Screen House, Seed Multiplication, crossing\_block\_trial, Specialty Trial +\item + plot\_width: plot width in meters +\item + plot\_length: plot length in meters +\item + field\_size: field size in hectares +\end{itemize} + +\hypertarget{deleting-trial-data}{% +\section{Deleting Trial Data}\label{deleting-trial-data}} + +To delete a trial data, click on the ``Delete trial data'' section. There are links to delete traits, layout and trial entry data. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_delete_trait1} \end{center} + +To delete assayed trait data, click on ``Delete trait data'' link. On the appeared dialog, confirm deletion by clicking on the ``Select Traits For Deletion'' button, then select one or more traits to delete from the trial. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_delete_trait2} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_detail_page_delete_trait3} \end{center} + +To delete trial layout data, click on the ``Delete layout data'' link. Confirm deletion on the appeared dialog. + +To Delete trial entry, click on ``Delete trial entry'' link. Confirm deletion on the appeared dialog. + +\hypertarget{managing-genotyping-plates}{% +\chapter{Managing Genotyping Plates}\label{managing-genotyping-plates}} + +Genotyping Plates represent the content of a genotyping plate sent to a genotyping facility (e.g.~samples in specific wells). To streamline this process, it is possible to upload this information or let the database create a plate for you. Once the genotyping plate is saved in the database it is then possible to export the information directly to genotyping facilities that are BrAPI compliant. The genotyping facility can then provide status information to us via BrAPI. + +To begin go to Manage-\textgreater Genotyping Plates. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials} \end{center} + +Here the genotyping plates are divided by Breeding Program. These sections can be expanded by clicking on one. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_expand} \end{center} + +\hypertarget{adding-a-new-genotyping-plate}{% +\section{Adding a New Genotyping Plate}\label{adding-a-new-genotyping-plate}} + +To begin, click on ``Add Genotyping Plate''. Notice that this form is split into three sections: ``Plate Information'', ``Well Information'', and ``Confirm''. The first section is for defining information about the genotyping plate, such as a Plate identifier, plate format (96 well), etc. The second section is for defining the samples in the wells, such as sample names, sample concentrations, well position, etc. The final section is for Submitting the info. + +All fields in the Plate Information section are required. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_add_trial} \end{center} + +In the Well Information section you can choose to either 1) Upload an XLS or XLSX spreadsheet with your sample layout or 2) let the database create the sample layout. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_add_trial_wells} \end{center} + +If you choose to upload an XLS or XLSX spreadsheet, the Spreadsheet Template info requires the following: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_upload_template} \end{center} + +In either case, the sample identifier is generally a concatenation of Plate name and well position, e.g.~MyGenotypingTrial1\_A01. In either case, you need to provide a ``source\_observation\_unit\_name'' for each sample. This can be a tissue sample name, a plant name, a plot name, or an accession name; however, in any case, the identifier must already exist in the database. This allows us to link the sample in the well to specific field trial plots, or, plants, or tissue\_samples. If you only know which accession is in the well, you can use the accession name. + +In the final Confirm section you can decide whether to submit this information to the genotyping facility you selected. This requires that the genotyping facility is BrAPI compliant to work. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_add_trial_confirm} \end{center} + +\hypertarget{genotyping-plate-detail-page}{% +\section{Genotyping Plate Detail Page}\label{genotyping-plate-detail-page}} + +If you open a specific genotyping plate, it will take you to the detail page. Here you can see the Accessions used in the plate (if you created the trial and the source\_observation\_unit\_names you used were plots, this will still work because we know the accession of the plot or plant or tissue sample). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_detail_info} \end{center} + +Further down you can see a graphical representation of your plate with well positions. This can be 96 well or 384 well depending on your plate format. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_trials_detail_table} \end{center} + +\hypertarget{Using-Field-Book-with-BreedBase}{% +\chapter{Using Field Book and BreedBase for Data Collection and Storage}\label{Using-Field-Book-with-BreedBase}} + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/FB_Section_Header} \end{center} + +\hypertarget{introduction-1}{% +\section{Introduction}\label{introduction-1}} + +\textbf{Field Book} is an open-source Android application designed to streamline phenotypic data collection, replacing traditional paper field books and reducing transcription errors. + +\textbf{BreedBase} supports integration with Field Book by enabling users to generate properly formatted field layout and trait files, import collected data, and optionally exchange data via \textbf{BrAPI}. + +This section provides step-by-step guidance on preparing Field Book for data collection using trial data stored in BreedBase, including file generation, manual file import, and data export. It assumes that trial fields and trait definitions already exist in BreedBase and that users have access to an Android device. + +\href{https://fieldbook.phenoapps.org/\#/}{\emph{Field Book User Manual}} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{installing-field-book}{% +\section{Installing Field Book}\label{installing-field-book}} + +Field Book can be installed on an Android device via the Google Play Store or manually from GitHub. To ensure access to the latest features and fixes, we recommend checking for updates regularly. + +\hypertarget{option-1-downloading-from-the-google-play-store}{% +\subsection{Option 1: Downloading from the Google Play Store}\label{option-1-downloading-from-the-google-play-store}} + +Download the Field Book app directly from the \href{https://play.google.com/store/apps/details?id=com.fieldbook.tracker}{Google Play Store} or scan the QR code below. + +\begin{center}\includegraphics[width=0.25\linewidth]{assets/images/QR_FieldBook_PlayStore} \end{center} + +\href{https://scribehow.com/shared/Fieldbook_Installation_and_Initial_Setup__3E-cowQMRcOKhfRl_sqFDQ}{\textbf{Walkthrough: Installation and Setup}} + +\hypertarget{option-2-manual-installation-from-github}{% +\subsection{Option 2: Manual Installation from Github}\label{option-2-manual-installation-from-github}} + +\href{https://scribehow.com/shared/Installing_Field_Book_from_GitHub__M0qchs3uThGb_ms2nDRg3A}{\textbf{Walkthrough: Installing Field Book from GitHub}} + +\hypertarget{keeping-field-book-updated}{% +\subsection{Keeping Field Book Updated}\label{keeping-field-book-updated}} + +Regular updates are recommended to access new features and bug fixes. + +\href{https://scribehow.com/shared/Updating_the_Field_Book_App__qdTEMk1lSkadYj8zXgOYgA}{\textbf{Walkthrough: Updating the Field Book App}}. + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{preparing-field-layout-files-in-breedbase}{% +\section{Preparing Field Layout Files in BreedBase}\label{preparing-field-layout-files-in-breedbase}} + +Field Book requires a \textbf{field layout file} containing a unique identifier for each plot or plant to be observed. Optionally, columns for row, column, pedigree, or previous phenotype data may be included for context. + +BreedBase provides tools to generate layout files directly from trial metadata. These files can be exported and then uploaded to Field Book for data collection. + +\href{https://scribehow.com/shared/Creating_a_Field_Layout_File_for_Field_Book_in_Breedbase__FDBwW83OS66kHXIvVsP-1A?referrer=documents}{\textbf{Walkthrough: Creating a Field Layout File in BreedBase}} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{importing-field-layout-files-into-field-book}{% +\section{Importing Field Layout Files into Field Book}\label{importing-field-layout-files-into-field-book}} + +Field layout files can be imported into Field Book via cloud storage or directly from the device. + +\hypertarget{option-1-import-from-cloud-storage}{% +\subsection{Option 1: Import from Cloud Storage}\label{option-1-import-from-cloud-storage}} + +Requirements: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + A generated field layout file. +\item + Linked cloud service (e.g., Google Drive) on the device. +\item + File uploaded to the cloud account. +\end{enumerate} + +\href{https://scribehow.com/shared/Importing_Field_Layout_Files_to_Field_Book_from_Cloud_Storage__5JLV59I8TzCgDeTkUPbIdA}{\textbf{Walkthrough: Importing from Cloud Storage}} + +\hypertarget{option-2-import-from-local-storage-device}{% +\subsection{Option 2: Import from Local Storage Device}\label{option-2-import-from-local-storage-device}} + +Requirements: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + A generated field layout file. +\item + File transferred to the Android device's local storage. +\end{enumerate} + +\href{link\%20to\%20be\%20added}{\textbf{Walkthrough: Importing from Local Device}} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{adding-traits-to-field-book-using-breedbase}{% +\section{Adding Traits to Field Book Using BreedBase}\label{adding-traits-to-field-book-using-breedbase}} + +For exported data to be compatible with BreedBase, trait names in Field Book must match the exact names stored in the database. + +BreedBase supports generation of \textbf{trait files} containing properly formatted trait names, types, units, and metadata. + +\hypertarget{creating-trait-files-in-breedbase}{% +\subsection{Creating Trait Files in BreedBase}\label{creating-trait-files-in-breedbase}} + +\href{https://scribehow.com/shared/Creating_a_Field_Book_Trait_File_in_BreedBase__AQKYEuyXRzK1H1lRFm--3Q}{\textbf{Walkthrough: Creating Trait Files in BreedBase}} + +\hypertarget{importing-trait-files-to-field-book}{% +\subsection{Importing Trait Files to Field Book}\label{importing-trait-files-to-field-book}} + +\href{https://scribehow.com/shared/Importing_Trait_Files_into_Field_Book__LuVCpgtfQCeDtWEl7xbZVw}{\textbf{Walkthrough: Importing Trait Files}} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{collecting-and-exporting-phenotype-data}{% +\section{Collecting and Exporting Phenotype Data}\label{collecting-and-exporting-phenotype-data}} + +This section describes the process for recording and exporting phenotype data using the Field Book application. It assumes that the required field layout and trait files have already been imported into the app. + +Users will be guided through entering phenotypic observations in the field and exporting collected data in a format compatible with downstream processing and upload to \textbf{BreedBase}. This standardized workflow helps ensure data integrity, traceability, and seamless integration with breeding databases. + +\hypertarget{collecting-phenotype-data-in-field-book}{% +\subsection{Collecting Phenotype Data in Field Book}\label{collecting-phenotype-data-in-field-book}} + +Once field layout and trait files are loaded, data collection can begin. Field Book allows users to view each plot or plant entry, record trait values in structured forms, and automatically timestamp and geotag observations (if device settings allow). + +\href{https://scribehow.com/shared/Collecting_Data_in_Fieldbook__hIsZkb9xSnKcLW65gN60lA}{\textbf{Walkthrough: Collecting Data in Field Book}} + +\hypertarget{exporting-phenotype-data-from-field-book-and-importing-to-breedbase}{% +\subsection{Exporting Phenotype Data from Field Book and Importing to BreedBase}\label{exporting-phenotype-data-from-field-book-and-importing-to-breedbase}} + +After data collection is complete, Field Book allows users to export data as a .csv file. This file can then be uploaded to BreedBase for storage and analysis. + +\href{https://scribehow.com/shared/Exporting_Phenotype_Data_in_FieldBook__ToUuuA3YTYGclEg7SDArLg}{\textbf{Walkthrough: Exporting Phenotype Data from Field Book}} + +Once exported, the .csv file can be imported into BreedBase through the trial's data upload interface. Proceed with uploading phenotype observations as described in the \textbf{Managing Phenotypic Data} section. + +It is important that: + +\begin{itemize} +\tightlist +\item + Trait names match those defined in the database. +\item + Plot or plant identifiers align with the original layout. +\end{itemize} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{using-brapi-for-data-transfer-between-field-book-and-breedbase}{% +\section{Using BrAPI for Data Transfer Between Field Book and BreedBase}\label{using-brapi-for-data-transfer-between-field-book-and-breedbase}} + +Field Book supports integration with \textbf{BreedBase} via the \textbf{BrAPI (Breeding API)} specification. BrAPI enables direct exchange of trial metadata and phenotype observations between Field Book and BreedBase without the need for manual file uploads. + +This walkthrough covers: + +\begin{itemize} +\tightlist +\item + Connecting Field Book to a BrAPI-enabled BreedBase server +\item + Importing trial layout and traits from BreedBase to Field Book +\item + Exporting collected phenotype data from Field Book to BreedBase +\end{itemize} + +\textbf{Prerequisites:} + +\begin{itemize} +\tightlist +\item + You must have a BreedBase user account with permissions to access trial and trait data. +\item + Your BreedBase instance must be configured with working BrAPI endpoints. +\item + The Field Book app must be installed on an Android device with internet access. +\end{itemize} + +\textbf{Advantages of Using BrAPI:} + +\begin{itemize} +\tightlist +\item + Reduces manual errors from file handling. +\item + Supports real-time syncing of data. +\item + Enables remote team coordination and efficient workflows. +\end{itemize} + +\hypertarget{connecting-field-book-to-a-brapi-server}{% +\subsection{Connecting Field Book to a BrAPI Server}\label{connecting-field-book-to-a-brapi-server}} + +\href{https://scribehow.com/shared/Enable_BrAPI_for_Field_Book_using_the_Scannable_BrAPI_Configuration__SUXsC8xKQX-rLEG-4H6BIA}{\textbf{Walkthrough: Connecting Field Book to a BreedBase BrAPI Server}} + +\hypertarget{importing-trials-and-traits-from-breedbase}{% +\subsection{Importing Trials and Traits from BreedBase}\label{importing-trials-and-traits-from-breedbase}} + +\href{https://scribehow.com/shared/Import_Field_Trials_from_BreedBase_via_BrAPI__R4munb3fQ-OO5f-WAZRnKg}{\textbf{Walkthrough: Importing Trials from BreedBase via BrAPI}} + +\href{https://scribehow.com/shared/Import_Traits_from_BreedBase_via_BrAPI__aqvmCO8QS_eziilPrntOoA?referrer=documents}{\textbf{Walkthrough: Importing Traits from BreedBase via BrAPI}} + +\hypertarget{collecting-data}{% +\subsection{Collecting Data}\label{collecting-data}} + +Proceed with entering phenotype observations as described in the \textbf{Collecting Phenotype Data in Field Book} section. + +Ensure that: + +\begin{itemize} +\tightlist +\item + Trait values match expected formats (e.g., numeric, categorical). +\item + Observations are saved regularly during field use. +\end{itemize} + +\hypertarget{exporting-phenotype-data-to-breedbase-via-brapi}{% +\subsection{Exporting Phenotype Data to BreedBase via BrAPI}\label{exporting-phenotype-data-to-breedbase-via-brapi}} + +\href{https://scribehow.com/shared/Exporting_Data_with_BrAPI__PaWGH98ESQWGrXIZk5lJTQ}{\textbf{Walkthrough: Sending Phenotype Data with BrAPI}} + +\hypertarget{verifying-data-in-breedbase}{% +\subsection{Verifying Data in BreedBase}\label{verifying-data-in-breedbase}} + +After exporting data to BreedBase via BrAPI, it is important to verify that the export was successful directly in BreedBase. + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Log in to your BreedBase instance. +\item + Navigate to the Manage Trials or Trial Detail page for the uploaded study. +\item + Confirm that: + + \begin{itemize} + \tightlist + \item + Trait data appears in the Phenotype Data tab. + \item + Data points align with expected plot/plant identifiers. + \item + Trait units and scales match those defined in the database. + \end{itemize} +\end{enumerate} + +\hypertarget{troubleshooting}{% +\subsection{Troubleshooting}\label{troubleshooting}} + +\begin{itemize} +\tightlist +\item + Connection fails: Ensure you have the correct BrAPI server URL and valid credentials. +\item + Traits do not load: Confirm that the trait exists in BreedBase. +\item + Upload fails: Check for internet connectivity, correct user permissions, and trait formatting errors. +\end{itemize} + +\hypertarget{managing-phenotypic-data}{% +\chapter{Managing Phenotypic Data}\label{managing-phenotypic-data}} + +To facilitate uploading process for phenotypic data, ``Manage Phenotypic Data'' page provides two options for uploading: Field Book Phenotype file in database format and phenotype file in Excel (.xls or .xlsx) file format. To access ``Manage Phenotypic Data'' page, clicking on ``Phenotyping'' in the ``Manage'' menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image178} \end{center} + +\hypertarget{uploading-fieldbook-phenotypes}{% +\section{Uploading Fieldbook Phenotypes}\label{uploading-fieldbook-phenotypes}} + +\hypertarget{export-field-book-database-file}{% +\subsection{Export Field Book Database File}\label{export-field-book-database-file}} + +The database upload of Field Book phenotype data relies on the ``Database'' format from the Field Book. Please make sure to export the ``Database'' format from the Field Book if you intend to upload the data using the Field Book Upload we describe below. If you prefer to use the ``Table'' format that the Field Book exports, you can modify this format to work with the Speadsheet Upload we describe below. + +\hypertarget{upload-field-book-database-file}{% +\subsection{Upload Field Book Database File}\label{upload-field-book-database-file}} + +To upload a Field Book Phenotype file in a database format, click the ``Upload Fieldbook'' link + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image185} \end{center} + +The ``Upload Fieldbook'' link on this page and ``Upload'' link on the ``Field Book Tools'' page open the same dialogue. Please follow instructions for uploading phenotypic files on the \ref{using-fieldbook-app} page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image112} \end{center} + +\hypertarget{uploading-spreadsheet-phenotypes}{% +\section{Uploading Spreadsheet Phenotypes}\label{uploading-spreadsheet-phenotypes}} + +To upload a phenotype file in an Excel (.xls or .xlsx) file format, click the ``Upload Spreadsheet'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image288} \end{center} + +Please specify ``Data Level'' (Plots or Plants) and select the Excel file that you want to upload. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image68} \end{center} + +\hypertarget{generating-spreadsheet-file}{% +\subsection{Generating Spreadsheet File}\label{generating-spreadsheet-file}} + +You can find more file format information by clicking on ``Spreadsheet Format'' link. Clicking on ``Spreadsheet Format'' will open the following dialog. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/phenotype_spreadsheet_upload_help_dialog} \end{center} + +Clicking on ``Create Phenotyping Spreadsheet'' will bring up a dialog where you can indicate the trial(s) you are interested in and the trait list you are interested in. Clicking ``Submit'' will download the xlsx file onto your computer, where you can then fill in the phenotypes. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/create_phenotype_spreadsheet_dialog} \end{center} + +\hypertarget{uploading-spreadsheet-file}{% +\subsection{Uploading Spreadsheet File}\label{uploading-spreadsheet-file}} + +To ensure that the file has a correct format for uploading, click on the ``Verify'' button. This will check the contents of the file and also perform quality checks on the values in the file. These checks include checking the trait definition for categorical values, minimum and maximum values, and data type checking. It will also check if there are already values uploaded for the given observation units and traits. If there are, there is an option to overwrite the existing values with the new values in your file. If the file is valid, only then can you click ``Store'' to store the information in the database. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/phenotype_spreadsheet_upload_dialog} \end{center} + +\hypertarget{managing-high-dimensional-phenotyping-data}{% +\chapter{Managing High Dimensional Phenotyping Data}\label{managing-high-dimensional-phenotyping-data}} + +\hypertarget{managing-transcriptomic-data}{% +\section{Managing Transcriptomic Data}\label{managing-transcriptomic-data}} + +\hypertarget{uploading-transcriptomic-data}{% +\subsection{Uploading Transcriptomic Data}\label{uploading-transcriptomic-data}} + +To upload transcriptomic data, go to the transcriptomics page by clicking ``Transcriptomics'' under the Manage tab. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/transcriptomics_menu} \end{center} + +Click the ``Upload Transcriptomics Data'' button to open the upload workflow dialog. On the second step ``Samples'', you will be prompted to make sure that your samples are already in the database. You can refer to the ``Managing Tissue Samples'' chapter for instructions on how to create a sampling trial. After creating a sampling trial or confirming that your samples exist, move on to the ``Protocol Info'' step. Here you can select an existing protocol, or create a new protocol. To create a protocol, click the ``Protocol Not Shown. Create a New Protocol'' button. From here you can fill in information about your new protocol and when finished, click ``Go to Next Step''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/create_transcriptomics_protocol} \end{center} + +You will need to have two csv files when uploading transcriptomics data. The first is the data matrix file, which should have these headers: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/data_matrix_headers} \end{center} + +The names in the sample\_name column should match the sample names you created in your sampling trial file. The header marked ``transcript\_name\_columns'' should be replaced with the transcript names you want to upload, each in its own column in the header and the respective expression value below. Example: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/data_matrix_example} \end{center} + +The second file is the transcript details file, which should have these headers: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/transcript_details_headers} \end{center} + +Each row under the transcript\_name column should contain each transcript name from your data matrix file. For example: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/transcript_details_example} \end{center} + +Once these two files are uploaded, click verify to check that they are formatted correctly and that the samples exist. If there are no errors, you can then click store to store your transcriptomics data. + +\hypertarget{downloading-transcriptomics-data}{% +\subsection{Downloading Transcriptomics Data}\label{downloading-transcriptomics-data}} + +To download your data, click the ``Uploaded Transcriptomics Data'' dropdown menu on the Transcriptomics page. Then click the ``Your Uploaded Transcriptomic Data'' dropdown. From here you will be able to see the transcriptomics data that you have uploaded. Click the ``Download'' button next to the data that you wish to download. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/download_transcriptomics} \end{center} + +\hypertarget{managing-barcodes}{% +\chapter{Managing Barcodes}\label{managing-barcodes}} + +SGN databases provide tools for generating barcodes for stock identification. To access ``Barcode Tools'' page, clicking on ``Barcodes'' in the ``Manage'' menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image82} \end{center} + +``Barcode Tools'' page provides four options for generating barcodes: + +\begin{itemize} +\item + Single barcode +\item + Multiple barcodes +\item + Plot phenotyping barcodes +\item + Trial barcodes +\end{itemize} + +To generate single barcode, clicking on ``Generate Barcode'' link on the ``Barcode Tools'' page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image274} \end{center} + +In the ``Generate Barcode'' section, specify the name of the barcode, size of the barcode, then clicking on ``Generate Barcode'' + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image245} \end{center} + +The database will generate a barcode for your stock. The barcode can be printed for your stock identification. It also appears on its corresponding stock page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image264} \end{center} + +If you have a list of stocks that you want to generate barcodes, you can use ``Download Stock Barcodes'' section. You have three options for entering stock names: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\item + Typing in stock names, or copy and paste from other file into the box (1) +\item + Choosing a list of stocks from your ``Lists'' (2), and transferring the list into the box (1) by clicking on ``paste'' button. +\item + Uploading a ``Tab-delimited Text File'' with stock names. +\item + Select an optional printing format from the available formats. +\end{enumerate} + +You can select printer settings that you prefer in the ``Printer Settings'' section. After you enter stock names and specify printer settings, clicking on ``Download Barcodes'' button at the bottom of the page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image248} \end{center} + +If you have a list of plots that you want to generate phenotyping barcodes, you can use ``Download Plot Phenotyping Barcodes'' section. You have three options for entering plot names: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\item + Typing in plot names, or copy and paste from other file into the box (1) +\item + Choosing a list of plots from your ``Lists'' (2), and transferring the list into the box (1) by clicking on ``paste'' button. +\item + Uploading a ``Tab-delimited Text File'' with plot names. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/plot_pheno_barcode} \end{center} + +If you have a list of trials that you want to generate barcodes, you can use ``Download Trial Barcodes'' section. You have three options for entering trial names: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\item + Typing in trial names, or copy and paste from other file into the box (1) +\item + Choosing a list of trial from your ``Lists'' (2), and transferring the list into the box (1) by clicking on ``paste'' button. +\item + Uploading a ``Tab-delimited Text File'' with trial names. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_barcode} \end{center} + +\hypertarget{using-the-label-designer}{% +\chapter{Using the Label Designer}\label{using-the-label-designer}} + +Breedbase provides an interactive design tool for creating custom labels. To access the label design tool, click on ``Label Designer'' in the ``Manage'' menu. The following sections explain your many options as you advance through each step of the design workflow. + +\hypertarget{select-a-data-source}{% +\section{Select a Data Source}\label{select-a-data-source}} + +The first step is to select a data source. To do so, you may directly select a specific data source from the ``Data Source'' dropdown menu (either a field trial, genotyping trial, or crossing trial) to populate your labels with the trial design information. Alternately, from the same dropdown menu, you may select a list to populate your labels with the list contents. Prior to making a selection from the ``Data Source'' dropdown menu, you may also filter the data sources by using the ``Datatype'' dropdown menu. You must then choose a level (plot, plant, etc.) before proceeding, using the ``Label for Every'' dropdown menu. To generate plot-level labels for more than one trial at once, select a list of trials as the source and plot as the level. Once you have made your selections, click the `Next' button to move to the next step in the workflow. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/gifs/gif1} \end{center} + +\hypertarget{set-page-and-label-size}{% +\section{Set Page and Label Size}\label{set-page-and-label-size}} + +After selecting the data source, you must choose whether to create a new design or load a saved design. If you choose the ``New'' selector, you will be prompted to select a page size and label size. If you do not see your page size or label size as an option, then select ``Custom'' from the relevant dropdown menu and enter your desired dimensions in pixels (each pixel is 1/72nd of an inch). If you choose the ``Saved'' selector, you will be prompted to select a saved design. After selecting a saved design, you will be taken directly to the design step with the saved design elements pre-loaded. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/gifs/gif2} \end{center} + +\hypertarget{design-your-label}{% +\section{Design Your Label}\label{design-your-label}} + +Having set the page and label formats, you will be presented with a label canvas where you can begin adding elements to your label. Select a type, field, size, and font, then click ``Add''. You can add text to an existing field or create a completely custom field by clicking ``Create Custom Field''. Once added, you can drag and drop elements, or you can delete them by clicking on the red box in their upper left corners. Barcodes can also be resized by dragging the green box in their lower right corners. If you are creating labels for a trial, it is highly recommended that you include a barcode encoding your plot, plant, or tissue sample names. These are the unique identifiers that will need to be included with any phenotypic or genotypic measurements loaded into the database. When you are satisfied with your label design, click ``Next''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/gifs/gif3} \end{center} + +\hypertarget{adjust-formatting-save-and-download}{% +\section{Adjust Formatting, Save, and Download}\label{adjust-formatting-save-and-download}} + +In the last step of the workflow, you can tweak your formatting and page layout, save your design, or download your labels. The ``Additional Settings'' dialog will allow you to adjust the page margins and margins between labels. The units are pixels (each pixel is 1/72nd of an inch). It is not recommended that you change these settings until you have already printed a test page. You can also set the number of copies per label, filter by replicate, or download only the first page for test purposes. To save your design, type a unique name in the text field and click ``Save''. This will save your design to your list manager, where you can make it public in order to share it with others. Finally, if you are ready, click download to generate and download your labels! + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/gifs/gif4} \end{center} + +\hypertarget{managing-downloads}{% +\chapter{Managing Downloads}\label{managing-downloads}} + +You can download phenotype, trial meta-data, pedigree, GBS genotype and GBS genotype QC files from the database to your computer by using ``Lists''. To download, clicking on ``Download'' in the ``Manage'' menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image255} \end{center} + +For each category, you can select a list of accessions from your ``Lists'' to download their phenotypes, pedigree, GBS genotype, GBS genotype QC. In the case of downloading trial meta-data, you would provide a list of trials, while for downloading phenotype and GBS genotype QC, you can also use a list of trials or traits. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/trial_meta_data_download} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image287} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image182} \end{center} + +\hypertarget{managing-odk-data-collection}{% +\chapter{Managing ODK Data Collection}\label{managing-odk-data-collection}} + +To access this page go to Manage and then ODK Data Collection. ODK is used for remotely collecting data on Android and IOS devices. We currently are working to support two ODK service providers, namely ONA and SMAP. We are using ONA to collect crossing information, including all lab activities following seed production. We are using SMAP for phenotypic data collection. + +\hypertarget{ona-crossing-information}{% +\section{ONA Crossing Information}\label{ona-crossing-information}} + +\hypertarget{managing-ona-crossing-information}{% +\subsection{Managing ONA Crossing Information}\label{managing-ona-crossing-information}} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/odk_ona_management} \end{center} + +To begin collecting data using the ONA ODK form you must first have a crossing plan in the form of a Cross Wishlist. To do this from this page, click the ``Export Cross Wishlist to ONA'' button. Please refer to the ``Create Cross Wihlist'' help section for more information. It is possible to view the current available cross wishlists by clicking the ``Export Cross Wishlist to ONA'' button and then clicking ``Available Cross Wishlists''. + +Once your cross wishlist is available, you can use your mobile ODK application to record crosses being done realtime. You can also record all laboratory activities following seed extraction up to greenhouse plantlet hardening. + +As you collect data using your mobile ODK application, your responses will be synchronized with our database. The ``Schedule Import for Selected Form'' section gives you options to perform the import daily or more frequently. It is also possible to initiate a data import from ONA at anytime by clicking ``Import Crossing Data from Selected Form on ONA''. + +\hypertarget{reviewing-plant-status}{% +\subsection{Reviewing Plant Status}\label{reviewing-plant-status}} + +The mobile ODK application has options to collect information about the status of plants in the field, such as if they are flowering. Images for each plant can also be recorded. The database will report this information here in a summary table that looks like the following. Notice that images are also transferred to the database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/odk_ona_cross_status} \end{center} + +\hypertarget{graphical-summary-for-performed-crosses}{% +\subsection{Graphical Summary For Performed Crosses}\label{graphical-summary-for-performed-crosses}} + +There is a section to summarize activities done for each cross. In this table each row represents a single cross performed. All the activities that have been performed will be shown here, such as ``first pollination'' and ``embryo rescue''. The scatter plot shown tracks seed numbers generated on the Y axis and date of activity on the X axis. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/odk_ona_cross_progress_graph} \end{center} + +\hypertarget{summary-information-for-performed-crosses}{% +\subsection{Summary Information For Performed Crosses}\label{summary-information-for-performed-crosses}} + +There is a secondary section to summarize what has been done across the entire Cross Wishlist. This tree structure shows all activities performed for a cross and shows how these crosses relate to the Cross Wishlist. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/odk_ona_cross_progress_tree} \end{center} + +\hypertarget{managing-tissue-samples}{% +\chapter{Managing Tissue Samples}\label{managing-tissue-samples}} + +To access this page go to Manage and then Tissue Samples. + +\hypertarget{tissue-samples-from-field-trials}{% +\section{Tissue samples from field trials}\label{tissue-samples-from-field-trials}} + +A field trial contains plots planted with a specific accession. Each plot can contain many plants, which in turn can contain many tissue samples. On the manage tissue sample page we can see the field trials that contain tissue samples already. We can choose to download the tissue sample layout as seen in the below picture. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_download} \end{center} + +If the field trial you want to collect tissue samples from is not in the above table, you can click the button highlighted below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_button} \end{center} + +Once you have clicked this button, you will enter a workflow that begins with the following introduction. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_intro} \end{center} + +Once you click next, you will need to select your trial. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_select} \end{center} + +Next, if your trial currently only has plot entries saved, you will be asked to enter how many plants are in each plot. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants} \end{center} + +Finally you will be asked how many tissue samples you want for each plant. You can specify a string to include in the tissue sample name, such as leaf or root. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissues} \end{center} + +Afterwards you should see the following success message, indicating that the tissue samples are saved. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_field_trial_submit} \end{center} + +\hypertarget{genotyping-plate-tissue-samples-96-or-384-well-plates}{% +\section{Genotyping Plate Tissue Samples (96 or 384 well plates)}\label{genotyping-plate-tissue-samples-96-or-384-well-plates}} + +A genotyping plate represents a 96 or 384 well plate. You can use the Coordinate Android application to create your plate layout, or you can upload your own Excel plate layout, or you can use the database to generate a plate layout. Ideally, you will use tissue sample names originating from a field trial as the ``source'' for each well tissue sample, but you can also use plant names, plot names, or accession names. + +From the manage tissue samples page, you can see the genotyping plates saved in the database. You can also download the layouts as shown below. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_genotyping_trial_download} \end{center} + +If you need to create a new genotyping plate, you can click the button shown below. This will guide you through a workflow for uploading or creating the new plate layout. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_tissue_samples_create_tissues_genotyping_trial_button} \end{center} + +Genotyping vendors require you to send a plate layout during submission. You can download the plate layout as shown above, or you can go to a genotyping plate detail page to download the Intertek formatted file. + +In the future you will be able to directly export your genotyping plate plate layout to vendors. + +\hypertarget{managing-observation-variables}{% +\chapter{Managing Observation Variables}\label{managing-observation-variables}} + +\hypertarget{managing-observation-variables-with-traits-methods-and-scales}{% +\section{Managing Observation Variables with Traits, Methods, and Scales}\label{managing-observation-variables-with-traits-methods-and-scales}} + +Observation variables are the identifiers used when collecting phenotypic data. An observation variable is composed of a trait, a method, and a scale. The trait describes the attribute being measured e.g.~`Plant Height'. The method defines the protocol in which the trait was observed e.g.~`Using a one meter long measuring stick'. The scale defines the units or dimensions for which the measurement was taken e.g.~`Meters'. + +Generally, observation variables are defined in ontologies that are predefined. We often use ontologies from cropontology.org. In this case, you will not be able to define your own observation variables directly; instead, you will need to contact us and we will add the observation variable for you. + +For databases where the user has greater control, we have an interface to allow addition of observation variables, along with traits, methods, and scales. To begin, go to the Search-\textgreater Traits page. + +If the database you are on allows you to directly add observation variables, you will see the following button at the bottom of the page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_start_button} \end{center} + +When you click the button, the following workflow will appear. You should be logged in or else it will not allow addition of the observation variable. The workflow begins with an introduction. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_intro} \end{center} + +On the next workflow step, you select the ontology that you want to insert the new observation variable into. You must also give a name and a definition for the new observation variable. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_observation_variable} \end{center} + +On the next workflow step, you select the trait ontology to use. Once you select a trait ontology, a select containing all the terms in the selected ontology will appear. You can either select a trait or if it does not exist in the select, you can create a new one by giving a name and a definition for the new trait. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_trait} \end{center} + +On the next workflow step, you select the method ontology to use. Once you select a method ontology, a select containing all the terms in the selected ontology will appear. You can either select a method or if it does not exist in the select, you can create a new one by giving a name and a definition for the new method. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_method} \end{center} + +On the next workflow step, you select the scale ontology to use. Once you select a scale ontology, a select containing all the terms in the selected ontology will appear. You can either select a scale or if it does not exist in the select, you can create a new one by giving a name and a definition for the new scale. You can also define a format, minimum, maximum, categories, and default value for the new scale. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_scale} \end{center} + +On the last page of the workflow, you confirm the submission. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_observation_variables_workflow_submit} \end{center} + +Afterwards, you can use the newly created observation variable ontology term in your phenotyping. + +\hypertarget{managing-image-data}{% +\chapter{Managing Image Data}\label{managing-image-data}} + +\hypertarget{image-phenotyping-dashboard}{% +\section{Image-Phenotyping Dashboard}\label{image-phenotyping-dashboard}} + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Upload raw image-captures in a compressed file (.zip) for orthophotomosaic assembly or upload previously stitched orthophotomosaic raster (.PNG, .JPG) imagery. +\item + Dashboard shows all field trials and uploaded imaging events in collapsible sections. +\item + Follow standard processes to manually create templates for assignment of plot-polygon images to the field experiment design. +\item + All imagery is shown with the spectral category within collapsible sections. Figure shows NIR imagery. +\item + Apply Fourier transform filtering, thresholding, and vegetation index masking. Plot-polygon images for all image processes are shown. +\item + Extract and export phenotypic values from plot-polygon images for analyses and model training. +\end{enumerate} + +\hypertarget{image-input}{% +\section{Image Input}\label{image-input}} + +Clicking ``Upload Imagery'' will open the following dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_intro} \end{center} + +Raw-captures can be uploaded in a compressed (.zip) file so that they can be assembled into an orthophotomosaic. If orthophotomosaic assembly is not required, raster images (.PNG, .JPG) can be uploaded. Example data is given for raw Micasense RedEdge 5-band multispectral captures and for stitched orthophotomosaics. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_field_trial} \end{center} + +To begin uploading images, a field trial must be selected. The field trial must already be saved in the database. For information about adding a field trial, please read the Field Trial documentation. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_drone_run} \end{center} + +The image data is added to an imaging (drone run) event. Here you can select a previously saved imaging event or you can create a new one by defining a name, description, and date. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_image_info_1} \end{center} + +The uploaded data can be raw image-captures or complete raster images. Here you can select whether orthophotomosaic stitching is required. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_image_info_2} \end{center} + +In the case that orthophotomosaic stitching is required, select `yes'. On the next step you will see the following: Upload a zipfile with the raw-captures. When uploading Micasense RedEdge raw-captures, provide images of the Micasense calibration panels in a zipfile as well. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_images_zipfile} \end{center} + +In the case that orthophotomosaic assembly is not required, simple upload the raster images. Select the number of image bands that will be uploaded e.g.~for a five band multispectral camera, select 5. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_image_info_3} \end{center} + +In the caes that orthophotomosaic stitching is not required, select `no'. On the next step you will see the following: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_upload_images_rasters} \end{center} + +Upload an image at each band with a unique name, description, and spectral type. + +\hypertarget{standard-process}{% +\section{Standard Process}\label{standard-process}} + +Once imagery is uploaded, it will appear on the dashboard under the field trial. Clicking the ``Run Standard Process'' button will begin extracting plot-polygon phenotypes from the imagery. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_button} \end{center} + +Clicking the button will open the following dialog. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_intro} \end{center} + +Select a drone run band to use in this process. In the case of the Micasense 5 band multispectral camera there will be 5 bands shown here; select the NIR channel in this case because it has the highest contrast. In the case of standard color images, there will only be the RGB Color Image option here. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_drone_run_band} \end{center} + +Rotate the image so that there the plots are oriented in a grid fashion. There can be a skew in the field layout, as seen in the following example. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_rotate} \end{center} + +Perform a rough cropping of the image by clicking on the four corners of the field. Cropping is important to remove any extraneous parts of the image. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_cropping} \end{center} + +This step shows a histogram of the cropped image. The standard process will magnitude threshold the top and low ends of the distribution. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_threshold_hist} \end{center} + +In this step, the template for the plot polygons in the experimental field design are associated to the image. First, defined the number of rows and columns in the field experiment. Then click the four corners of the image, in respect to the top right, top left, botton left, and bottom right positions. Next click on ``Draw Plot Polygon Template''. Review the template and clear/repeat the process until the template matches well. It is possible to ``copy/paste'' templates in the case where there are large breaks in the field design. Next, scroll down to the ``assign Plot Polygons to Field Trial Entities'' section. Select the location of Plot Number 1 as either ``top left'' or ``top right'' and whether the field design is serpentine or zigzag. Click on ``Generate Assignments'' and review that the names of the plots appear correctly in the overlay on the image. Finally, click ``Finish and Save Polygons to Plots'' when you have have confirmed the assignments. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_plot_polygon} \end{center} + +Next, the dialog shows you that the standard process will be repeated for all uploaded image bands. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_apply} \end{center} + +Next, choose which vegetation indices to apply. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_vegetation_index} \end{center} + +Next, choose the phenotypic values to extract. You must define the time point for which the phenotype is; if the field trial has a planting date, the time point will automatically be populated as image date minus the planting date. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_image_phenotyping_standard_process_phenotypes} \end{center} + +After completing the standard process, the job will continue in the background until it completes. You can check the status of the job from the dashboard. + +\hypertarget{ground-control-points}{% +\section{Ground Control Points}\label{ground-control-points}} + +Ground control points can be saved after an imaging event has undergone the standard process on orhomosaics. Ground control points can then be used across imaging events on the same field experiment in order to automate the entire standard process. + +\hypertarget{managing-phenotypic-image-data}{% +\chapter{Managing Phenotypic Image Data}\label{managing-phenotypic-image-data}} + +\hypertarget{uploading-image-files}{% +\section{Uploading Image files}\label{uploading-image-files}} + +Clicking ``Phenotyping'' under Manage, and then ``Upload Images'' will open this dialog: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image_upload_dialog} \end{center} + +If uploading images directly, select the file format ``Images''. A single image or multiple selected images can be uploaded using this file format. Each of these images must have been downloaded from the Fieldbook app, or have a filename that follows this structure: observationUnitName, traitname, number, timestamp joined by underscores. The filename can not include any underscores other than the ones separating each value. For example: 38873\_branching\_5\_2025-07-28.jpg + +If the images have been taken in the Fieldbook app, they will have EXIF data that includes their associated stocks and traits. This EXIF data will be automatically parsed and the associations will be made when the images are uploaded. + +After selecting your images for upload, clicking verify will check if your filenames or image metadata is correct and corresponds to existing stocks in the database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/filename_verify_success} \end{center} + +If the verification is successful, click store to save these images in the database. + +If uploading images with associated phenotypic data, select the file format ``Images with Associated Phenotypes''. A spreadsheet with the phenotypic values needs to be uploaded alongside a zipfile of images. The spreadsheet can be in .xls or .xlsx format, and the column headers need to be: + +observationUnitName \textbar{} observationVariableName \textbar{} value \textbar{} timestamp \textbar{} image\_name \textbar{} person + +The image\_name column should match the file names of the associated images in the zipfile. + +\hypertarget{uploading-images-directly-to-stocks}{% +\section{Uploading Images Directly to Stocks}\label{uploading-images-directly-to-stocks}} + +In the Images section on a stock details page you can add a new image associated with that stock by clicking the ``Add Image'' button. This will associate any images uploaded through this dialog with the corresponding stock. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/stock_add_image} \end{center} + +\hypertarget{managing-vcf-data}{% +\chapter{Managing VCF Data}\label{managing-vcf-data}} + +\hypertarget{uploading-vcf-data}{% +\section{Uploading VCF Data}\label{uploading-vcf-data}} + +Genotyping data in VCF can be loaded from the web-interface. Breedbase can store any genotypic variants from a VCF, allowing for polyploids, structural variants, etc. without problems. + +To begin go to Manage-\textgreater Genotyping Plates and click the button seen below: Note that you do not need to have genotyping plates uploaded to upload VCF data; you may upload genotyping data to accessions or you can upload genotyping data for tissue samples in genotyping plates. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_upload_button} \end{center} + +The workflow begins with an intro: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_upload_dialog_intro} \end{center} + +On the following step in the workflow, a genotyping project is defined or selected. A genotyping project is a high-level entity for grouping several genotyping events. It is defined with a name, description, name, breeding program, and genotyping facility (IGD, Intertek, etc.). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_upload_dialog_project} \end{center} + +The following step is to define or select a genotyping protocol. A genotyping protocol represents the set of markers being called against a specific reference genome. A genotyping protocol is defined with a name, description, reference genome name, species name, and a location of data generation. Note in the picture that you can select whether the samples in your file are accessions or tissue samples in the database; tissue samples are for when a genotyping plate is stored in the database. There is an option to parse the sample names for appended sequencing numbers from IGD, where the sample names are like ``accession:igdnumber''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_upload_dialog_protocol} \end{center} + +The final step is to select the VCF from your computer and upload it. The web interface can be used to upload files arbitrarily large; it is a NGINX configuration to set this size. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_upload_dialog_vcf} \end{center} + +\hypertarget{searching-and-downloading-vcf-data}{% +\section{Searching and Downloading VCF Data}\label{searching-and-downloading-vcf-data}} + +The Search Wizard is the primary means of querying data in the database. Go to Search-\textgreater Wizard to begin. + +Once genotyping protocols are stored, select Genotyping Protocols from the first dropdown menu. Then if you select one or more and select Accessions from the second dropdown menu, you will see the accessions for which genotypes were stored. As seen in the following picture, there is a section for filtering genotypes by chromosome, start position, and end position. Genotypes can be downloaded in VCF or DosageMatrix formats. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_wizard_download_prot} \end{center} + +Using the ``Default genotyping protocol'' which is configured in a system, you can query over field phenotypic evaluations before downloading genotypes and phenotypes. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_genotyping_data_wizard_trial} \end{center} + +\hypertarget{searching-protocols}{% +\section{Searching Protocols}\label{searching-protocols}} + +Genotyping protocols can be search by going to Search-\textgreater Genotyping Protocols. To download genotypes accessions must be selected, though any combination of search criteria can be used to filter and select those accessions. If a genotyping protocol is not selected, then the default genotyping protocol set in the configuration will be used. Genotyping protocols can also be selected in the wizard. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_wizard_genotyping_protocols_download_main} \end{center} + +The genotyping download menu on the Search Wizard presents options for filtering by chromosome, start position, and end position. Genotypes can be downloaded in VCF of Dosage Matrix formats. The genomic relationship matrix (GRM) can be downloaded for the selected accessions in a tab-delimited matrix format or in a three-column format that is useful in Asreml. Genotypes can be computed from the parents in the pedigree if those parents are genotyped by clicking on the ``compute from parents'' checkbox. Additionally, the GRM can be computed using genotypes of parents in the pedigree if the ``compute from parents'' checkbox is selected. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/search_wizard_genotyping_protocols_download_genotypes} \end{center} + +As is described elsewhere, the Search Wizard presents a way to filter phenotypic values by minimum and maximum values, and allow for download in CSV and Excel formats. + +\begin{center}\includegraphics[width=0.75\linewidth]{assets/images/search_wizard_genotyping_protocols_download_phenotypes} \end{center} + +\hypertarget{detail-pages-and-deletion}{% +\section{Detail Pages and Deletion}\label{detail-pages-and-deletion}} + +The genotyping protocol detail page will show all information about the protocol such as the reference genome used, the header information lines in the uploaded VCF file, the markers involved, and the samples genotyped. + +The markers section will show all markers used and their annotations, such as position, chromosome, alternate allele, reference allele, marker format, etc. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/genotyping_protocol_detail_page_markers} \end{center} + +The samples section will show all samples genotyped. Notice the Download links in the table which can be used to easily get the VCF file results for each genotyped samples with all markers in the genotyping protocol. For getting mulitple samples at once, use the Search Wizard as discussed above. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/genotyping_protocol_detail_page_samples} \end{center} + +The genotyping protocol and all associated genotyping data can be deleted from the genotyping protocol page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/genotyping_protocol_detail_page_delete} \end{center} + +\hypertarget{managing-spectral-data}{% +\chapter{Managing Spectral Data}\label{managing-spectral-data}} + +Breedbase has implemented a flexible spectral data storage protocol that handles spectral data irrespective of the source spectrometer. Spectral data storage and analysis in Breedbase makes use of the R package \emph{waves} for outlier identification, plotting, sample aggregation, and prediction model training. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/waves_breedbase_schema} \end{center} + +\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center} + +\hypertarget{upload-spectral-data}{% +\section{Upload Spectral Data}\label{upload-spectral-data}} + +Spectral data can be added as a CSV file that includes metadata in the leftmost columns followed by one column per spectral measurement to the right. Rows represent a single scan or sample, each with a unique ID that must match to a Breedbase observationUnitName. Future data transfer using \href{https://brapi.org}{BrAPI} will allow for interoperability with data collection software. + +To upload a spectral dataset, navigate to the `Manage NIRS Data' page by selecting `NIRS' in the `Manage' menu and click the blue `Upload NIRS' button. This will open an upload workflow. A link to the required file format and an example .csv file can be found by clicking in the light blue info box in this workflow. Another example of the file format is shown below. + +\begin{itemize} +\tightlist +\item + \textbf{id}: Optional identifier for each NIRS read. The id must be an integer. +\item + \textbf{sampling\_id}: Optional identifier for each sample. Strings are allowed. +\item + \textbf{sampling\_date}: Optional field. The format allowed is: YYYY-MM-DD. +\item + \textbf{observationunit\_name}: Required field that matches existing data in the database. It can be the plot name, subplots, plant name, or tissue sample, depending how your trial is designed. +\item + \textbf{device\_id}: Optional field to identify your device. Strings are allowed. +\item + \textbf{device\_type}: Required field. It is possible upload data for a single device type. They can be: SCiO, QST, Foss6500, BunchiN500, or LinkSquare. +\item + \textbf{comments}: Optional field for general comments. All other columns are required wavelengths. You can add how many columns you want upload -- there is no limit. +\end{itemize} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_main_display} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/upload_NIRS_example_format} \end{center} + +\hypertarget{evaluate-and-remove-outliers}{% +\section{Evaluate and Remove Outliers}\label{evaluate-and-remove-outliers}} + +Spectral calibration models can be heavily affected by the presence of outliers, whether they come from spectrometer spectral artifacts or user errors. Mahalanobis distance (Mahalanobis, 1936) is a measure of the distance between a single observation and a larger distribution and is commonly used in the identification of outliers in a multivariate space (Des Maesschalck et al, 2000). The \emph{FilterSpectra()} function in the R package \href{https://CRAN.R-project.org/package=waves}{\emph{waves}} calculates the Mahalanobis distance of each observation in a given spectral matrix using the \emph{stats::mahalanobis()} function. Observations are identified as outliers if the squared distance is greater than the 95th percentile of a \(\chi\)\textsuperscript{2}-distribution with \emph{p} degrees of freedom, where \emph{p} is the number of columns (wavelengths) in the spectral matrix (Johnson and Wichern, 2007). In Breedbase, this procedure is applied on a per-dataset basis on upload and outliers are given binary tags ``Outlier.'' + +\hypertarget{plot-spectra}{% +\section{Plot Spectra}\label{plot-spectra}} + +After outlier identification, a plot is generated using the \emph{PlotSpectra()} function in \href{https://CRAN.R-project.org/package=waves}{\emph{waves}}. This function uses the filtered spectra and \emph{ggplot2::ggplot()} to create a line plot with outliers highlighted by color. A list of rows identified as outliers are shown beneath the plot. Plots are saved as .png files and linked to the original input datasets. Plot image files can be downloaded via the ``Download Plot'' button in the upload workflow. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/upload_NIRS_plot_example} \end{center} + +\hypertarget{aggregate-spectra}{% +\section{Aggregate Spectra}\label{aggregate-spectra}} + +To obtain a stable and reliable spectral profile, most spectrometer manufacturers recommend that multiple spectral scans are captured for each sample. While some spectrometers aggregate these scans internally, many do not, requiring the user to do so before analysis can take place. Breedbase handles these cases upon data upload following filtering steps by calling the \emph{AggregateSpectra()} function from \href{https://CRAN.R-project.org/package=waves}{\emph{waves}}, saving the aggregated scans for future access through the search wizard feature. Scans are aggregated by sample mean (e.g.~plot-level basis) according to the provided observationUnitName field. After aggregation, the user exits the upload workflow and the raw data file is saved in the upload archive. + +\hypertarget{references}{% +\section{References}\label{references}} + +\begin{itemize} +\tightlist +\item + De Maesschalck, R., Jouan-Rimbaud, D., and Massart, D. L. (2000). The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. +\item + Johnson, R. A. \& Wichern, D. W. (2007). Applied Multivariate Statistical Analysis (6th Edition). p 773. +\item + Mahalanobis, P. C. (1936). On the generalized distance in statistics. National Institute of Science of India. +\end{itemize} + +\protect\hyperlink{spectral-analysis}{\textbf{Analysis tool documentation}} + +\hypertarget{managing-sequence-metadata}{% +\chapter{Managing Sequence Metadata}\label{managing-sequence-metadata}} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/sequence_metadata_manage} \end{center} + +\hypertarget{what-is-sequence-metadata}{% +\section{What is Sequence Metadata?}\label{what-is-sequence-metadata}} + +Sequence Metadata is a feature that allows for the efficient storage and retrieval of sequence annotations for a specific region along a reference genome. The annotation data can contain a primary ``score'' value and any number of secondary key/value attribute data. For example, Sequence Metatadata can store MNase open chromatin scores for every 10 basepairs along the reference genome as well as genome-wide association study (GWAS) statistics, including the trait information associated with the result. This data can then be filtered by position and/or scores/attribute values and even cross-referenced with markers stored in the database. + +\hypertarget{loading-sequence-metadata}{% +\section{Loading Sequence Metadata}\label{loading-sequence-metadata}} + +Sequence Metadata can be loaded into the database using a gff3-formatted file. The following columns are used to load the data: + +\begin{itemize} +\tightlist +\item + \textbf{\#1 / seqid:} The name of the database feature (ie chromosome) the metadata is associated with (The feature name must already exist as a feature in the database) +\item + \textbf{\#4 / start:} The metadata's start position +\item + \textbf{\#5 / end:} The metadata's end position +\item + \textbf{\#6 / score:} (optional) The primary score attribute of the metadata +\item + \textbf{\#9 / attributes:} (optional) Secondary key//value attributes to be saved with the score. These should be formatted using the gff3 standard (key1=value1;key2=value2). The attribute key cannot be either score, start, or end. +\end{itemize} + +To upload the gff3 file: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + Go to the \textbf{Manage \textgreater{} Sequence Metadata} page +\item + Click the \textbf{Upload Sequence Metadata} button +\item + On Step 2 of the Wizard, select the Type of data to be uploaded + + \begin{itemize} + \tightlist + \item + This groups similar datasets together in the same Data Type category + \end{itemize} +\item + On Step 3 of the Wizard, select an existing Protocol or create a new one + + \begin{itemize} + \tightlist + \item + The Protocol is used to describe how the data was generated and define the score value and any secondary attributes. Adding the attributes (and their descriptions) to the Protocol will allow the Sequence Metadata queries to filter the data based on the value of one or more of these attributes. Attributes not defined in the Protocol will still be stored and displayed on retrieval, but will not be able to be used in a search filter. + \end{itemize} +\item + Finally, select and upload your gff3 file to the database. The database will verify the format of the file before its contents are stored. +\end{enumerate} + +\hypertarget{searching-sequence-metadata}{% +\section{Searching Sequence Metadata}\label{searching-sequence-metadata}} + +To retrieve stored Sequence Metadata, go to the \textbf{Search \textgreater{} Sequence Metadata} page. + +\hypertarget{basic-search}{% +\subsection{Basic Search}\label{basic-search}} + +The basic Sequence Metadata search options include selecting the reference genome and species, the chromosome, and (optionally) the start and/or end position(s) along the reference genome. In addition, one or more specific protocols can be selected to limit the results. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/sequence_metadata_search_basic} \end{center} + +The Sequence Metadata search results are returned as a table, including the chromosome and start/stop positions of the annotation, along with the primary score value and any additional key/value attributes. The markers column will include a list of marker names of any stored markers that are found within the start/stop positions of the Sequence Metadata. The data can be downloaded as a table in an Excel or CSV file or a machine-readable (code-friendly) JSON file. If the Sequence Metadata JBrowse configuration is set, the filtered results can be displayed as a dynamic JBrowse track. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/sequence_metadata_search_results} \end{center} + +\hypertarget{advanced-search}{% +\subsection{Advanced Search}\label{advanced-search}} + +Any number of advanced search filters can be applied to the query. The advanced filters can limit the search results by the value of the primary score and/or any of the secondary attribute values. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/sequence_metadata_search_advanced} \end{center} + +\hypertarget{marker-integration}{% +\section{Marker Integration}\label{marker-integration}} + +A table of Sequence Metadata annotations are embedded on the Marker/Variant detail page. The table will include any annotations that span the poisiton of the marker (for data of the same reference genome and species). + +\hypertarget{sequence-metadata-api}{% +\section{Sequence Metadata API}\label{sequence-metadata-api}} + +A publicly accessible RESTful API (Application Programming Interface) is available to query the database for Sequence Metadata directly from your programming environment (R, python, etc) to be used in analysis. The data is returned in a JSON format. Documentation for the API can be found on the \textbf{Manage \textgreater{} Sequence Metadata} page + +\hypertarget{managing-outliers-in-dataset}{% +\chapter{Managing Outliers in Dataset}\label{managing-outliers-in-dataset}} + +\hypertarget{what-is-outliers-functionality-in-dataset}{% +\section{What is Outliers Functionality in Dataset ?}\label{what-is-outliers-functionality-in-dataset}} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/outliers_dataset_icon} \end{center} + +As in step \protect\hyperlink{search-wizard}{The Search Wizard} we can create a dataset. + +The dataset incorporates a feature to identify outlier points, which we may choose to exclude from a specific dataset. It's important to note that these exclusions only apply at the dataset level, and no data is permanently removed from the database. Additionally, outlier categorization can be modified at any time, and these changes are visible to all other functionalities within the system. + +Each dataset stores a wholly unique set of outlier points, completely independent of any other dataset in the database. Outliers are specifically designated for traits within datasets, exclusively encompassing phenotype data. If a particular dataset lacks traits as a part of wizard selection, this functionality is not available. + +Each trait has its own set of defined outliers. + +\hypertarget{accessing-trait-visualization}{% +\section{Accessing Trait Visualization}\label{accessing-trait-visualization}} + +Once you've selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/outliers_dataset_basic_panel} \end{center} + +\hypertarget{interpreting-visual-elements}{% +\section{Interpreting Visual Elements}\label{interpreting-visual-elements}} + +Once you've selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. + +\begin{itemize} +\tightlist +\item + \textbf{Green Points}: As per the legend, represent values for the selected trait that fall below the cut-off point set by the slider. (non-outliers) +\item + \textbf{Black Outlined Points}: These data points are outlined with black borders, indicating that they are currently designated as outliers in the database. +\item + \textbf{Red Points}: The red data points denote the cut-off points established by the slider for the allowable deviation value. +\end{itemize} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/outliers_dataset_full_visualisation} \end{center} + +\hypertarget{choosing-cut-off-values}{% +\section{Choosing Cut-Off Values}\label{choosing-cut-off-values}} + +You have two fundamental options for setting cut-off points: + +\begin{itemize} +\tightlist +\item + \textbf{Median with MAD}: This option involves using the median (middle value) along with the Mean Absolute Deviation (MAD) as a reference point for determining cut-off values. +\item + \textbf{Mean with Standard Deviation}: Alternatively, you can choose to use the mean (average) in conjunction with the Standard Deviation to set cut-off points. +\end{itemize} + +\hypertarget{setting-deviation-multiplier}{% +\section{Setting Deviation Multiplier}\label{setting-deviation-multiplier}} + +The slider allows you to specify the deviation multiplier from a central point, which influences the cut-off values. + +\hypertarget{utilizing-graph-controls}{% +\section{Utilizing Graph Controls}\label{utilizing-graph-controls}} + +Beneath the graph, you'll find four buttons, each serving a distinct function: + +\begin{itemize} +\tightlist +\item + \textbf{Add selection to outliers}: This button enables you to save the current cut-off points to the database for future reference. +\item + \textbf{Reset outliers for current trait}: You can use this option to reset outliers for the selected trait. +\item + \textbf{Reset all outliers}: This button allows you to reset outliers for the entire dataset. +\item + \textbf{Download Phenotype Table without outliers}: You can download the phenotype data table in a comma-separated value format file, using this feature, with outliers excluded for selected dataset. +\end{itemize} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/outliers_dataset_actions} \end{center} + +These tools and functions are designed to provide you with control and insights when working with data visualization and outliers. + +\hypertarget{data-analysis-tools}{% +\chapter{Data Analysis Tools}\label{data-analysis-tools}} + +SGN databases provides several tools for phenotype data analysis, marker-assisted selection, sequence and expression analyses, as well as ontology browser. These tools can be found in the ``Analyze'' menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image114} \end{center} + +\hypertarget{selection-index}{% +\section{Selection Index}\label{selection-index}} + +To determine rankings of accessions based on more than one desirable trait, SGN databases provide a ``Selection Index'' tool that allows you to specify a weighting on each trait. To access the tool, clicking on ``Selection Index'' in the ``Analyze'' menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image251} \end{center} + +On the Selection Index page, selecting a trial that you want to analyze. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image95} \end{center} + +After you selected a trial, you can find traits that were assayed in that trial in the ``Trait'' box. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image78} \end{center} + +Selecting a trait that you want to include in the analysis will open a new dialogue showing the selected trait and a box that you can assign a ``Weight'' of that trait. After you are done, you can continue by selecting another trait by clicking on ``Add another trait'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image304} \end{center} + +After you selected another trait, this page will automatically update information for you by showing all of the traits that you selected for the analysis. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image76} \end{center} + +You also have options to choose a reference accession, choose to include accessions with missing phenotypes, scaling values to a reference accession. After you complete your setting, clicking on ``Calculate Rankings'' + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image343} \end{center} + +The Selection Index tool will generate rankings of accessions based on the information that you specified. You can copy the results to your system clipboard, convert the table data to CSV format, or print the data. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image326} \end{center} + +Clicking on ``Raw Average'' will display average values of the phenotypes of those ranked accessions. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image150} \end{center} + +Selection Index tool also allows you to save top ranked accessions directly to ``Lists''. You can retrieve top ranked accessions by selecting a number or a percent. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image156} \end{center} + +\hypertarget{genomic-selection}{% +\section{Genomic Selection}\label{genomic-selection}} + +The prediction of breeding values for a trait is a one step or two steps process, depending on what stage in your breeding cycle you are. The first step is to build a prediction model for a trait using a training population of clones with phenotype and genotype data. If you have yet to select parents for crossing for your first cycle of selection you can use the breeding values of the training population. If you are at later stages of your selection program, you need to do the second step which is applying the prediction model on your selection population. All clones in your training and selection populations must exist in the database. + +To use the genomic selection tool, on \href{http://cassavabase.org/}{\emph{cassavabase.org}}, select ``Genomic Selection'' from the ``analyze'' pull-down menu. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image247} \end{center} + +There are three ways to build a model for a trait. + +\hypertarget{method-1}{% +\subsection{Building a Model - Method 1:}\label{method-1}} + +One way to build a model is, using a trait name, to search for trials in which the trait was phenotyped and use a trial or a combination of trials to build a model for the trait. For example, if you search for ``mosaic disease severity, you will get a list of trials you can use as training populations. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image160} \end{center} + +You will get a list of trials (as shown below) in which the trait of your interested was phenotyped. From the list, you can use a single trial as a training population or combine several trails to form a training population for the prediction model of the trait. Let''s say, you want to create a training population using individuals from trials ``cassava ibadan 2001/02'' and ``cassava ibadan 02/03'' and build a model for ``cassava mosaic disease severity'' using all clones from the training population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image249} \end{center} + +Select the trials to combine (the same coloured), click `done selecting', click the ``combine trials and build model'' button, and you will get a model and its output for the trait. On the model detail page, you can view the description of input data used in the model, output from the model and search interface for selection populations the model you can apply to predict their breeding values. The description of the input data for the model includes the number of phenotyped clones, and the number of markers, scatter and frequency distribution plots for the phenotype data, relationship between the phenotype data and GEBVs, population structure. The model output includes model parameters, heritability of the trait , prediction accuracy, GEBVs of the individuals from the training population and marker effects. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image330} \end{center} + +Expand each section to see detailed information. + +If you expand the `Trait phenotype data' section, you will find plots to explore the phenotype data used in the model. You can assess the phenotype data using a scatter and histogram plots and the descriptive statistics. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image244} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image263} \end{center} + +A regression line between observed phenotypes and GEBVs shows the relationship between the two. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image83} \end{center} + +You can also explore if there is any sub-clustering in the training population using PCA. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image93} \end{center} + +To check the model accuracy, a 10-fold cross-validation test, expand the `model accuracy' section. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image328} \end{center} + +Marker effects are also available for download. To do so, expanad the `Marker Effects' section and click the `Download all marker effects' link and you will get a tab delimited output to save on your computer. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image74} \end{center} + +The breeding values of the individuals used in the training population are displayed graphically. Mousing over each data point displays the clone and its breeding value. To examine better, you can zoom in into the plot by selecting an area on the plot. You can download them also by following the ``Download all GEBVs'' link. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image147} \end{center} + +\hypertarget{estimating-breeding-values-in-a-selection-population}{% +\subsubsection*{Estimating breeding values in a selection population}\label{estimating-breeding-values-in-a-selection-population}} + + +If you already have a selection population (in the database), from the same model page, you can apply the model to the selection population and estimate breeding values for all the clones in the population. You can search for a selection population of clones in the database using the search interface or you can make a custom list of clones using the \protect\hyperlink{working-with-lists}{\emph{list interface}}. If you click the ``search for all relevant selection populations'', you will see all relevant selection populations for that model. However, this option takes long time decause of the large set of populations in the database and the filtering. Therefore, the fastest way is to search for each of your selection populations by name. If you are logged in to the website you will also see a list of your custom set of genotyped clones. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image338} \end{center} + +To apply the model to a selection population, simply click your population name or ``Predict Now'' and you will get the predicted breeding values. When you see a name of (or acronym{]}) of the trait, follow the link and you will see an interactive plot of the breeding values and a link to download the breeding values of your selection population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image334} \end{center} + +\hypertarget{method-2}{% +\subsection{Building a Model - Method 2}\label{method-2}} + +Another way to build a model is by selecting a trial, instead of selecting and searching for a specific trait. This approach is useful when you know a particular trial that is relevant to the environment you are targeting to breed material for. This method allows you to build models and predict genomic estimated breeding values (GEBVs) for several traits within a single trial at once. You can also calculate selection index for your clones when GEBVs are estimated for multiple traits. + +To do this select the ``Genomic Selection'' link found under the ``analyze'' menu. This will take you to the same home page as used with Method 1. However, instead of entering information to search for in ``Search for a trait'', click on ``Use a trait as a trial population''. This will expand a new menu that will show all available trials. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image344} \end{center} + +\begin{center}\includegraphics[width=0.25\linewidth]{assets/images/image329} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image341} \end{center} + +To begin creating the model, select the existing trial that you would like to use. In this example I will be using the trial and trait data from ``Cassava Ibadan 2002/03'' trial. Clicking on a trial will take you to a page where you can find information such as number of markers and number of phenotypes clones. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image322} \end{center} + +In addition to the number of phenotype clones and number of markers, the main page for the trial selected also has information and graphs on phenotypic correlation for all of the traits. By moving your cursor over the graph you can read the different values for correlation between two traits. A key with all of the trait names of the acronyms used can be found in the tab below the graph. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image151} \end{center} + +Below the ``Training population summary'' there is a tab for ``Traits''. Clicking on this tab will show all available traits for the specific trial. You can create a model by choosing one or multiple traits in the trial and clicking ``Build Model''. In this example, the traits for ``cassava bacterial blight severity'' and ``cassava mosaic disease severity'' have been selected. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image69} \end{center} + +Clicking on ``Build Model'' will take you to a new page with the models outputs for the traits. Under the ``Genomic Selection Model Output'' tab you can view the model output and the model accuracy. Clicking on any of the traits will take you to a page with information about the model output on that individual trait within the trial. There you can view all of the trait information that was seen in more detail in \protect\hyperlink{method-1}{\emph{Method 1}}. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image336} \end{center} + +You can apply the models to simultaneously predict GEBVs for respective traits in a selection population by clicking on ``Predict Now'' or the name of the selection population. You can also apply the models to any set of genotyped clones that you can create using the ``lists'' feature. For more information on lists, click \protect\hyperlink{working-with-lists}{\emph{here}}. Follow the link to the trait name to view and download the predicted GEBVs for the trait in a selection population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image171} \end{center} + +To compare clones based on their performance on multiple traits, you can calculate selection indices using the form below. Choose from the pulldown menu the population with predicted GEBVs for the traits and assign relative weights for each trait. The relative weight of each trait must be between 0 - 1. 0 being of least weight and importance, not wanting to consider that particular trait in selecting a genotype and 1 being a trait that you give highest importance. + +In this example we will be using the ``Cassava Ibadan 2002/03'' population and assigning values to each of the traits. Remember that there is a list of acronyms and trait names at the bottom of the page for reference. After entering whatever values you would like for each trait click on the ``Calculate'' button to generate results. This will create a list of the top 10 genotypes that most closely match the criteria that you entered. The list will be displayed right below the ``selection index'' tab. This information can also be downloaded onto your computer by clicking on the ``Download selection indices'' link underneath the listed genotypes and selection indices. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image81} \end{center} + +\hypertarget{building-a-model---method-3}{% +\subsection{Building a Model - Method 3}\label{building-a-model---method-3}} + +In addition to creating a model by searching for pre-existing traits or by preexisting trial name, models can also be created by using your own list of clones. This creates a model by using or creating a training population. + +The page to use the third Method for creating a population model is the same as for the other two models. Select ``Genomic Selection'' from under the ``analyze'' menu of the main toolbar. This will take you to the Genomic Selection homepage and show you all three available methods to create a model. To see and use Method 3 scroll down and click on the tab labeled ``Create a Training Population''. This will open a set of tools that will allow you to use pre-existing lists or to create a new list. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image138} \end{center} + +Once the ``Create a Training Population'' tab is opened you have the option to use a pre-existing list or create new one. To learn how to create a list, click \protect\hyperlink{working-with-lists}{\emph{here}}. The ``Make a new list of plots'' link will take you directly to the Search Wizard that is usually used to create lists. + +Please note: the only lists that can be used in Method 3 to create a model are lists of plots and trials. If the pre-existing list is not of plots or trials (for example, traits, or locations) it will not show up and cannot be used as a training population. When you create you use a list of trials, the trials data will be combined to create a training data set. + +To use your custom list of plots or trials as a training population, select the list and click ``Go''. This will take you to a detail page for the training population. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image181} \end{center} + +From here on you can build models and predict breeding values as described in \protect\hyperlink{method-2}{\emph{Method 2}}\textbf{.} + +\hypertarget{genome-browsing}{% +\section{Genome Browsing}\label{genome-browsing}} + +There are two ways to evaluate genotype information within the browser, from an accession detail page or a trial detail page. + +\hypertarget{browsing-genotype-data-by-accession}{% +\subsection{Browsing Genotype data by Accession}\label{browsing-genotype-data-by-accession}} + +If you are interested in browsing genotype information for a single accession, for example `BAHKYEHEMAA', navigate to the accession detail page. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image152} \end{center} + +Near the bottom of the detail page is a collapsible section called ``Accession Jbrowse''. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image20} \end{center} + +This section will contain a link to the accession jbrowse page if the necessary genotype data is available. Clicking the link should take you to a page that looks like this, a which point you can browsre the genotype data in the form of a vcf track aligned to the latest build of the genome. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image318} \end{center} + +\hypertarget{browsing-genotype-data-by-trial}{% +\subsection{Browsing Genotype data by Trial}\label{browsing-genotype-data-by-trial}} + +If you are interested in browsing genotype information for the accessions within a given trial, navigate to the trial detail page. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image277} \end{center} + +Halfway down the page is a collapsible section called ``Trial Jbrowse''. This section will contain a link to the trial jbrowse page if the necessary genotype data for at least two accessions planted in the trial is available. + +\begin{center}\includegraphics[width=0.5\linewidth]{assets/images/image268} \end{center} + +Clicking the link should take you to a page that looks like this, a which point you can browse the genotype data in the form of vcf tracks aligned to the latest build of the genome. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/image327} \end{center} + +\hypertarget{principal-component-analysis-pca}{% +\section{Principal Component Analysis (PCA)}\label{principal-component-analysis-pca}} + +Principal component analysis helps estimate and visualize if there is sub-grouping of individuals within a dataset based on a number of variables. Currently, you can use marker data to run PCA on datasets. + +You can run PCA from multiple places on the website. To do PCA on + +\begin{enumerate} +\def\labelenumi{(\arabic{enumi})} +\item + individuals from a trial, go to the trial detail page and find the PCA tool under the ``Analysis tools'' section. +\item + individuals from a training population you used in a GS modeling, do your modeling and find the PCA tool in the model output page. +\item + individuals in a training population and selection population you applied the training model, do your modeling, apply the model on the selection population and find the PCA tool on the selection population prediction output page. +\item + individuals in a list of accessions you created, for example using the search wizard, go to the ``Analyze'' menu and select the ``Population Structure'', select your list of individuals and run PCA. +\item + individuals from multiple trials, create a list of the trials using the search wizard, go to the ``Analyze'' menu and select the ``Population Structure'', select your list of trials and run PCA. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/pca_iita_naccri_trials} \end{center} + +With all the options, you will get a interactive plot of the two PCs (shown below) that explain the largest variance. Point the cursor at any data point and you will see the individual name with its corresponding PCs scores. By clicking the `Download all PCs', you can also download the 10 PCs scores in the text format. + +\hypertarget{anova}{% +\section{ANOVA}\label{anova}} + +Currently, ANOVA is implemented for a single trial (single year and single location). You can do ANOVA for RCBD, CRD, Alpha and Augmented trial designs. ANOVA is done using linear mixed effects model, where the genotypes is fixed effect and the replications and blocks are random effects. Fixed effect significance level is computed using ``lmer'' from ``lmeTest'' R package. + +You can do ANOVA from two places: trial detail and training population detail. In both cases, if the phenotype data was from the supported trial designs, + +-- Go to the ANOVA section down in the trial or training population page + +-- Select the trait of you want to perform ANOVA + +-- Click the ``Run ANOVA'' and wait for the result + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/anova-dm} \end{center} + +\hypertarget{clustering-k-means-hierarchical}{% +\section{Clustering (K-Means, Hierarchical)}\label{clustering-k-means-hierarchical}} + +The K-Means method allows you to partition a dataset into groups (K number). The hierarchical clustering, agglomerative, allows you to explore underlying similarity and visualize in a tree structure (dendrogram) the different levels of similarities (clusters) among samples. You can do clustering based on marker data, phenotype data and GEBVs. When you use phenotype data, first clone averages for each trait are calculated. Both methods use Euclidean distance as a measure of similarity. For the hierachical clustering, the complete-linkage (farthest neighbour) method is used to link up clusters. + +There are three pathways to using this tool. + +\begin{enumerate} +\def\labelenumi{(\arabic{enumi})} +\tightlist +\item + When you have data in the form of a list or dataset from the search wizard: +\end{enumerate} + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\item + -- go to the ``Analyze'' menu and select the clustering option +\item + -- make sure you are logged in +\item + -- Select the relevant genotyping protocol, if you are clustering using genotype data +\item + -- select your list or dataset, click ``Go'' +\item + -- select clustering type +\item + -- select the data type to use +\item + -- If you are running K-Means clustering, provide the number of partitions (K). If left blank it will partition the data set into optimal numbers for the dataset. +\item + -- click the ``Run Cluster'' and wait for the analysis to finish or queue the request and wait for an email with the analysis result. +\item + -- You can download the outputs following the download links. +\end{enumerate} + +\begin{enumerate} +\def\labelenumi{(\arabic{enumi})} +\setcounter{enumi}{1} +\tightlist +\item + From the trial detail page: +\end{enumerate} + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\item + -- Go to the ``Analysis Tools'' section +\item + -- Follow steps D to G in (1) +\end{enumerate} + +\begin{enumerate} +\def\labelenumi{(\arabic{enumi})} +\setcounter{enumi}{2} +\tightlist +\item + In the solGS pipeline: +\end{enumerate} + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\tightlist +\item + -- Once you you are in a model output put page, you will see a section where you can do clustering in the same way as above (option 2). +\end{enumerate} + +K-Means clustering: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/k-means-cluster} \end{center} + +Hierarchical clustering: + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/hclustering} \end{center} + +\hypertarget{genetic-gain}{% +\section{Genetic Gain}\label{genetic-gain}} + +You can check for genetic gain by comparing the the GEBVs of a training and a selection population. You can do this in the solGS pipepline once you build a model and apply the model to predict the GEBVs of a selection population. Once at that stage, you will see a section ``Check Genetic Gain''. Select a selection population to compare with the training population and click the ``Check Genetic Gain'' button. The genetic gain will be visualized in boxplots. You can download the boxplot(s) as well as the GEBVs data used for the plot(s). + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/genetic-gain} \end{center} + +\hypertarget{kinship-and-inbreeding-coefficients}{% +\section{Kinship and Inbreeding Coefficients}\label{kinship-and-inbreeding-coefficients}} + +This tool allows you to estimate genetic relatedness between a pair of individuals (kinship), homozygousity across loci in an individual (inbreeding coefficient), and genetic similarity of an individual relative to the rest of the population (averge kinship). + +There are three pathways to using this tool. + +(1) When you have a list or dataset clones, created from the search wizard: + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\item + -- go to the ``Analyze'' menu and select the kinship and inbreeding +\item + -- make sure you are logged in +\item + -- Select the genotypic protocol for the marker data +\item + -- select your list or dataset of clones, click ``Go'' +\item + -- click the ``Run Kinship'' and wait for the analysis to finish, depending on the data size this may take minutes. You can choose to submit the analysis and wait for an email notice to view the results or wait for it to complete. +\item + -- You can download the output following the download links. +\end{enumerate} + +(2) From the trial detail page: + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\item + -- Go to the ``Analysis Tools'' section +\item + -- Follow steps C to G in (1) +\end{enumerate} + +(3) In the solGS pipeline: + +\begin{enumerate} +\def\labelenumi{(\Alph{enumi})} +\tightlist +\item + -- Once you you are in a model output put page, scroll down to the ``Kinship and Inbreeding'' section and run kinship. +\end{enumerate} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/kinship-inbreeding} \end{center} + +\hypertarget{creating-crossing-groups}{% +\section{Creating Crossing Groups}\label{creating-crossing-groups}} + +If you calculate selection index based on GEBVs of multiple traits, and you want to select a certain proportion of the indexed individuals (e.g.~top 10\%, or bottom 10\%) and then you want to partition the selected individuals into a number of groups based on their genotypes, you can use the k-means clustering method. + +The procedure is: + +\begin{enumerate} +\def\labelenumi{(\arabic{enumi})} +\item + predict GEBVs for 2 or more traits +\item + In the models output page, calculate selection indices. Note the name of the selection index data. +\item + Go to the clustering section, +\end{enumerate} + +-- select the selection index data, + +-- select ``K-means'', + +-- select ``Genotype'', + +-- in the K-numbers textbox, fill in the number of groups you want to create, + +-- in the selection proportion textbox, fill in the proportion of the indexed individuals you want to select, e.g.~for the top 15 percent, 15. if you wish to select bottom performing, prefix the number with minus sign (e.g.~-15) + +-- then run cluster and wait for the result. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/selection_proportion_clustering} \end{center} + +\hypertarget{search-wizard-genomic-relationship-matrix-grm-download}{% +\section{Search Wizard Genomic Relationship Matrix (GRM) Download}\label{search-wizard-genomic-relationship-matrix-grm-download}} + +The genomic relationship matrix (GRM) is useful for understanding underlying structure in your population. Breedbase can compute the GRM using rrBLUP. First, select accessions in the search wizard and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn\_local.conf). Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. The GRM can be returned in a matrix format (.tsv) which shows all pairwise relationships between the selected accessions and is useful for visualization; alternatively, the GRM can be returned in a three-column format (.tsv) which is useful for programs like ASReml outside of Breedbase. The GRM can also be returned as a simple correlation heatmap image (.pdf). The GRM can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox ``compute from parents''; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_wizard_genotype_analyses_grm} \end{center} + +\hypertarget{search-wizard-genome-wide-association-study-gwas}{% +\section{Search Wizard Genome Wide Association Study (GWAS)}\label{search-wizard-genome-wide-association-study-gwas}} + +Performing a genome wide association study (GWAS) can determine genotypic markers which are significantly correlated to phenotypic traits. Breedbase can compute GWAS using rrBLUP. First, select accessions and trait(s) in the search wizard, and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn\_local.conf). Several traits can be selected in the search wizard; if the traits are not to be treated as repeated measurements then select ``no'' in the select box and this will tell Breedbase to return GWAS results independently for the selected traits. If the selected traits are indeed all repeated measurements then select ``yes'' in the select box and Breedbase will return as single GWAS analysis across all the phenotypic records. Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. GWAS results can be returned in a tabular format (.tsv) where the -log10(p-values) for the selected traits are returned; alternatively, the GWAS results can be returned as Manhattan and QQ plots for the selected traits. The GWAS can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox ``compute from parents''; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. + +The GWAS will filter the data by the input MAF and missing data filters provided. After filtering the data is imputed using an ``EM'' method in rrBLUP. The Kinship matrix (GRM) is computed from the imputed genotypic data and used in the GWAS model. The GWAS uses fixed effects for different field trials and replicates in the phenotypic data. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_wizard_genotype_analyses_gwas} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_wizard_genotype_analyses_manhattan_plot} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/search_wizard_genotype_analyses_qq_plot} \end{center} + +\hypertarget{spectral-analysis}{% +\section{Spectral Analysis}\label{spectral-analysis}} + +Visible and near-infrared spectroscopy (vis-NIRS) can be related to reference phenotypes through statistical models to produce accurate phenotypic predictions for unobserved samples, increasing phenotyping throughput. This technique is commonly used for predicting traits such as total starch, protein, carotenoid, and water content in many plant breeding programs. Breedbase implements the R package \href{https://CRAN.R-project.org/package=waves}{\emph{waves}} to offer training, evaluation, storage, and use of vis-NIRS prediction models for a wide range of spectrometers and phenotypes. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/waves_breedbase_schema} \end{center} + +\hypertarget{dataset-selection}{% +\subsection{Dataset selection}\label{dataset-selection}} + +In order to initiate an analysis, the user must select one or more datasets using \ref{search-wizard}. A dataset in Breedbase can contain observationUnit-level (plot-, plant-, or sample-level) trial metadata and phenotypic data from one or more trials. After navigating to the ``NIRS'' webpage under the ``Manage'' tab in Breedbase, the user can initiate an analysis and select one of these datasets as input for model training. An optional test dataset can be selected in the second step of the workflow. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_prediction_workflow_intro} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_prediction_workflow_dataset} \end{center} + +\hypertarget{cross-validation}{% +\subsection{Cross-validation}\label{cross-validation}} + +Five cross-validation schemes that represent scenarios common in plant breeding are available for this analysis. These include CV1, CV2, CV0, and CV00 as outlined below and described in depth by Jarquín et al.~(2017) as well as random and stratified random sampling with a 70\% training and 30\% validation split. For those schemes from Jarquín et al.~(2017), specific input datasets must be chosen based on genotype and environment relatedness. Cross-validation choices: * \textbf{Random sampling} (70\% training / 30\% validation) * \textbf{Stratified random sampling}, stratified based on phenotype (70\% training / 30\% validation) * \textbf{CV1}, untested lines in tested environments * \textbf{CV2}, tested lines in tested environments * \textbf{CV0}, tested lines in untested environments * \textbf{CV00}, untested lines in untested environments + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_cv} \end{center} + +\hypertarget{preprocessing}{% +\subsection{Preprocessing}\label{preprocessing}} + +Preprocessing, also known as pretreatment, is often used to increase the signal to noise ratio in vis-NIR datasets. The \emph{waves} function \emph{DoPreprocessing()} applies functions from the \emph{stats} and \emph{prospectr} packages for common spectral preprocessing methods with the following options: * Raw data (default) * First derivative * Second derivative * Gap segment derivative * Standard normal variate (SNV; Barnes et al., 1989) * Savitzky-Golay polynomial smoothing (Savitzky and Golay, 1964) + +For more information on preprocessing methods and implementation, see the \href{https://CRAN.R-project.org/package=waves}{\emph{waves}} manual, available through CRAN: \href{https://cran.r-project.org/web/packages/waves/waves.pdf}{waves.pdf} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_snv} \end{center} + +\hypertarget{algorithms}{% +\subsection{Algorithms}\label{algorithms}} + +Several algorithms are available for calibration model development in Breedbase via the \href{https://CRAN.R-project.org/package=waves}{\emph{waves}} package. The \emph{TrainSpectralModel()} function in waves performs hyperparameter tuning as applicable using these algorithms in combination with cross validation and train functions from the package \emph{caret}. Currently, only regression algorithms are available, but classification algorithms such as PLS-DA and SVM clasification are under development. * \textbf{Partial least squares regression} (PLSR; Wold et al., 1982; Wold et al., 1984) is a popular method for spectral calibrations, as it can handle datasets with high levels of collinearity, reducing the dimensionality of these data into orthogonal latent variables (components) that are then related to the response variable through a linear model (reviewed in Wold et al., 2001). To avoid overfitting, the number of these components included in the final model must be tuned for each use case. The PLSR algorithm from the \emph{pls} package is implemented by waves. * \textbf{Random Forest regression} (RF; Ho, 1995) is a machine learning algorithm based on a series of decision trees. The number of trees and decisions at each junction are hyperparameters that must be tuned for each model. Another feature of this algorithm is the ability to extract variable importance measures from a fitted model (Breiman, 2001). In Breedbase, this option is made available through implementation of the RF algorithm from the package randomForest in the waves function TrainSpectralModel(). This function outputs both model performance statistics and a downloadable table of importance values for each wavelength. It is worth noting that this algorithm is computationally intensive, so the user should not be alarmed if results do not come right away. Breedbase will continue to work in the background and will display results when the analysis is finished. * \textbf{Support vector machine regression} (SVM; Vapnik, 2000) is another useful algorithm for working with high-dimension datasets consisting of non-linear data, with applications in both classification and regression. The package waves implements SVM with both linear and radial basis function kernels using the kernlab package. + +\hypertarget{output-common-model-summary-statistics}{% +\subsection{Output: common model summary statistics}\label{output-common-model-summary-statistics}} + +After training, model performance statistics are both displayed on a results webpage and made available for download in .csv format. These statistics are calculated by the \emph{TrainSpectralModel()} function in \href{https://CRAN.R-project.org/package=waves}{\emph{waves}} using the \emph{caret} and \emph{spectacles} packages. Reported statistics include: * Tuned parameters depending on the model algoritm * \textbf{Best.n.comp}, the best number of components to be included in a PLSR model * \textbf{Best.ntree}, the best number of trees in an RF model * \textbf{Best.mtry}, the best number of variables to include at every decision point in an RF model * \textbf{RMSECV}, the root mean squared error of cross-validation * \textbf{R2cv}, the coefficient of multiple determination of cross-validation for PLSR models * \textbf{RMSEP}, the root mean squared error of prediction * \textbf{R2p}, the squared Pearson's correlation between predicted and observed test set values * \textbf{RPD}, the ratio of standard deviation of observed test set values to RMSEP * \textbf{RPIQ}, the ratio of performance to interquartile distance * \textbf{CCC}, the concordance correlation coefficient * \textbf{Bias}, the average difference between the predicted and observed values * \textbf{SEP}, the standard error of prediction * \textbf{R2sp}, the squared Spearman''s rank correlation between predicted and observed test set values + +\hypertarget{export-model-for-later-use}{% +\subsection{Export model for later use}\label{export-model-for-later-use}} + +Once a model has been trained, it can be stored for later use. This action calls the \emph{SaveModel()} function from \href{https://CRAN.R-project.org/package=waves}{\emph{waves}}. Metadata regarding the training dataset and other parameters specified by the user upon training initialization are stored alongside the model object itself in the database. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_export_model} \end{center} + +\hypertarget{predict-phenotypes-from-an-exported-model-routine-use}{% +\subsection{Predict phenotypes from an exported model (routine use)}\label{predict-phenotypes-from-an-exported-model-routine-use}} + +For phenotype predictions, users select a dataset and can then choose from models in the database that were trained using the same spectrometer type as the spectral data in the chosen dataset. Predicted phenotypes are stored as such in the database and are tagged with an ontology term specifying that they are predicted and not directly measured. Metadata regarding the model used for prediction is stored alongside the predicted value in the database. Predicted phenotypes can then be used as normal in other Breedbase analysis tools such as the Selection Index and GWAS. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_select_model} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/manage_NIRS_prediction_results} \end{center} + +\hypertarget{faq}{% +\subsection{FAQ}\label{faq}} + +The Breedbase Spectral Analysis Tool does not allow for prediction models involving data from multiple spectrometer types at once. + +References * Barnes, R.J., M.S. Dhanoa, and S.J. Lister. 1989. Standard normal variate transformation and de-trending of near-infrared diffuse reflectance spectra. Appl. Spectrosc. 43(5): 772-777. doi: 10.1366/0003702894202201. * Breiman, L. 2001. Random forests. Mach. Learn. 45: 5-32. doi: 10.1201/9780429469275-8. * Ho, T.K. 1995. Random decision forests. Proc. Int. Conf. Doc. Anal. Recognition, ICDAR 1: 278-282. doi: 10.1109/ICDAR.1995.598994. * Jarquín, D., C. Lemes da Silva, R.C. Gaynor, J. Poland, A. Fritz, et al.~2017. Increasing Genomic-Enabled Prediction Accuracy by Modeling Genotype x Environment Interactions in Kansas Wheat. Plant Genome 10(2): plantgenome2016.12.0130. doi: 10.3835/plantgenome2016.12.0130. * Johnson, R.A., and D.W. Wichern. 2007. Applied Multivariate Statistical Analysis (6th Edition). De Maesschalck, R., D. Jouan-Rimbaud, and D.L. Massart. 2000. The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. doi: 10.1016/S0169-7439(99)00047-7. * Mahalanobis, P.C. 1936. On the generalized distance in statistics. Natl. Inst. Sci. India. * Savitzky, A., and M.J.E. Golay. 1964. Smoothing and Differentiation of Data by Simplified Least Squares Procedures. Anal. Chem. 36(8): 1627-1639. doi: 10.1021/ac60214a047. * Shrestha, R., L. Matteis, M. Skofic, A. Portugal, G. McLaren, et al.~2012. Bridging the phenotypic and genetic data useful for integrated breeding through a data annotation using the Crop Ontology developed by the crop communities of practice. Front. Physiol. 3 AUG(August): 1-10. doi: 10.3389/fphys.2012.00326. * Vapnik, V.N. 2000. The Nature of Statistical Learning Theory. Springer New York, New York, NY. * Wold, S., A. Ruhe, H. Wold, and W.J. Dunn, III. 1984. The Collinearity Problem in Linear Regression. The Partial Least Squares (PLS) Approach to Generalized Inverses. SIAM J. Sci. Stat. Comput. 5(3): 735-743. doi: 10.1137/0905052. * Wold, S., M. Sjöström, and L. Eriksson. 2001. PLS-regression: a basic tool of chemometrics. Chemom. Intell. Lab. Syst. 58(2): 109-130. doi: 10.1016/S0169-7439(01)00155-1. + +\hypertarget{general-mixed-model-tool}{% +\section{General Mixed Model Tool}\label{general-mixed-model-tool}} + +The general mixed model tool is available at /tools/mixedmodels and a link is provided from the Analyze menu. + +To use the mixed model tool, first create dataset using the Wizard containing the data that you would like to analyze. + +Select the Mixed Model tool from the Analyze menu. + +You are presented with a workflow. On the first step of the workflow, select the dataset that you wish to analyze, click on ``Choose dataset'' to continue. + +The second part of the workflow presents you with the traits in the dataset; you can select one or more traits from the lists using the select buttons. If you selected one trait, a bargraph of the trait distribution will be shown. Click the ``Next step'' button to move to the next screen. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/mixedmodel_tool_model_build_step} \end{center} + +On the model build screen, all the factors are displayed that are contained within the dataset. The factors are presented as a list of blue buttons that can be dragged using the mouse to areas on the screen which build a mixed model equation. The areas correspond to fixed factors, random factors, and optionally to more complex factors, such as fixed factors with interaction and fixe factors with vriable slope/intersects. Drag the available factors to the corresponding area. To calculate BLUPs for germplasm, drag the germplasmName button to the ``Random factors'' area. To calculate BLUEs, drag it to the ``Fixed factors'' area. The factors need to have different levels contained within them, for example, if there is only one trial in the dataset, it cannot be used as one of the factors. Click on ``Run analysis and got to next step'' to run the mixed model and display the results. + +The result view contains two tabs, one with the raw data, either BLUPS or BLUEs, and the other the adjusted means from the raw data. + +The results can be stored in the database as an analysis, by clicking the button provided on the top of the data. + +\hypertarget{genomic-prediction-of-cross-performance-gpcp}{% +\section{Genomic Prediction of Cross Performance (GPCP)}\label{genomic-prediction-of-cross-performance-gpcp}} + +The GPCP tool is available at /tools/gcpc and a link is provided from the Analyze menu. The GCPC tool implements genomic prediction with additive and directional dominance in the linear mixed model to predict for cross performance. + +Before using the tool, first create a dataset using the Wizard containing the data that you would like to analyze. (The dataset should have genotyping\_protocols). Second, create Selection Indices for your traits using Selection Index in Analyze Menu. + +To use the tool, Select the GPCP tool from the Analyze menu. + +Then, select the dataset with genotyping\_protocols that you wish to analyze, click on ``Proceed to Factor Selection'' to load available factors that can be included in the model. + +Select the factors you wish to include in the model either as Fixed or Random. Click ``None'' for factors that you don''t want to include in the model. Note that the ``germplasmName'' is factored as Random by default. + +The next step is to select the selection index for your traits on the dropdown menu. + +Once you are through, click ``Run GPCP'' to run the model. The output will be presented in form of a table with ``ID'', ``Parent1'', ``Parent2'' and their cross prediction merit organized in descending order. The results will also have sex information based on whether the dataset has plant sexes available in the database. + +\hypertarget{tool-compatibility}{% +\section{Tool Compatibility}\label{tool-compatibility}} + +The dataset definition enables one to predict whether the dataset can be used in various analysis tools. + +Upon creating a dataset, the site will automatically predict its compatibility with the available analysis tools and report these values on the dataset details page. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/tool_compatibility_details} \end{center} + +In the table, each tool will report to the user which traits are available to be analyzed based on phenotype data, and if different types of analyses are available, these will also be reported to the user. Some tools may give a warning sign to indicate that this dataset is compatible, but with potentially low sample sizes. Hover over the warning symbol to get a readout of the reason for the warning. + +Below the table, there is a button that enables the user to re-calculate tool compatibility. This can be useful if a dataset is created before phenotypes are uploaded to a trial, since phenotype data is used in determining dataset compatibility. Even if the page appears to hang, do not worry; the compatibility check will continue in the background, and you can check later. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/tool_compatibility_data_summary} \end{center} + +Below the tool compatibilities, there is also a summary of the data encompassed by the dataset and the criteria used for determining tool compatibility. Those criteria are used in the following way: - Correlation: A dataset can be used in a correlation analysis if there are many phenotype measurements for different traits made on the same accession. - Population Structure (PCA): A genotype PCA can be run if there are many accessions all genotyped with the same protocol. A phenotype PCA can be run if many accessions all have measurements on many traits. - Clustering: Like a PCA, clustering can be done in both phenotype and genotype modes. They have the same requirements as PCA. - Kinship \& Inbreeding: A dataset with many accessions genotyped with the same protocol can be used for kinship analyses. - Stability: A dataset containing many accessions with the same trait measured across multiple locations can be used in stability analyses. - Heritability: This requires one or more trials with the same trait measured on the same accession across those trial(s). - Mixed Models: This requires sufficient accession numbers, trait measurements, and trial designs. - GWAS: A dataset is compatible with GWAS if there are many accessions genotyped for the same genotyping protocol, and the genotyping protocol has enough markers to run a GWAS. In addition, each accession needs to be phenotyped for a trait. - Boxplotter: There must be sufficient trait measurements to make a boxplot of the trait. + +In addition to being on the dataset details page, tool compatibilities may be listed on the dataset selection screens for analysis tools. The compatibilities are non-blocking; you may always try using a dataset in an analysis even if there are warnings or if it is deemed non-compatible. As before, you can hover over the warning symbols to see why a dataset may not have statistical power. For analyses with multiple modes, such as clustering and PCA, you can also hover over the compatibliity checkmark to see what types (phenotype or genotype) the dataset is compatible with. + +\hypertarget{vector-viewer}{% +\chapter{Vector Viewer}\label{vector-viewer}} + +BreedBase provides a vector viewer tool that can generate, save, or upload vectors onto BreedBase. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/vectorViewer} \end{center} + +The vector viewer tool comes with many interactive features: + +\begin{itemize} +\item + Highlighting +\item + Location display +\item + Zooming in / panning out +\item + A feature and restriction enzyme table that support real time editing, deleting, or adding +\end{itemize} + +To highlight a region of the vector viewer, click and drag your cursor clockwise. To delete the new highlight, either start a new highlighted segment by clicking on the vector, or click the ``Update/Reset'' button. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/vectorHighlight} \end{center} + +To find the location of a spot on the vector, hover your cursor over the vector and a number will appear. That is the location, in base pairs, of your cursor. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/cursorLocation} \end{center} + +To zoom in and pan out, click the ``+'' or the ``-'' button underneath the vector. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/ZoomIn} \end{center} + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/ZoomOut} \end{center} + +To add a feature or restriction site, click ``Add''. To edit a feature or restriction site, select the feature or restriction site of choice, click ``Edit'', and make any wanted changes. To delete a feature or restriction site, selected the feature or restriction site that you want to delete and then click ``Delete''. + +\begin{center}\includegraphics[width=0.95\linewidth]{assets/images/vectorButtons} \end{center} + +\end{document} diff --git a/docs/Using-Field-Book-with-BreedBase.html b/docs/Using-Field-Book-with-BreedBase.html new file mode 100644 index 0000000000..3088023845 --- /dev/null +++ b/docs/Using-Field-Book-with-BreedBase.html @@ -0,0 +1,639 @@ + + + + + + + Chapter 12 Using Field Book and BreedBase for Data Collection and Storage | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 12 Using Field Book and BreedBase for Data Collection and Storage

+

+
+

12.1 Introduction

+

Field Book is an open-source Android application designed to streamline phenotypic data collection, replacing traditional paper field books and reducing transcription errors.

+

BreedBase supports integration with Field Book by enabling users to generate properly formatted field layout and trait files, import collected data, and optionally exchange data via BrAPI.

+

This section provides step-by-step guidance on preparing Field Book for data collection using trial data stored in BreedBase, including file generation, manual file import, and data export. It assumes that trial fields and trait definitions already exist in BreedBase and that users have access to an Android device.

+

Field Book User Manual

+
+
+
+

12.2 Installing Field Book

+

Field Book can be installed on an Android device via the Google Play Store or manually from GitHub. To ensure access to the latest features and fixes, we recommend checking for updates regularly.

+
+

12.2.1 Option 1: Downloading from the Google Play Store

+

Download the Field Book app directly from the Google Play Store or scan the QR code below.

+

+

Walkthrough: Installation and Setup

+ +
+
+

12.2.2 Option 2: Manual Installation from Github

+

Walkthrough: Installing Field Book from GitHub

+ +
+
+

12.2.3 Keeping Field Book Updated

+

Regular updates are recommended to access new features and bug fixes.

+

Walkthrough: Updating the Field Book App.

+ +
+
+
+
+

12.3 Preparing Field Layout Files in BreedBase

+

Field Book requires a field layout file containing a unique identifier for each plot or plant to be observed. Optionally, columns for row, column, pedigree, or previous phenotype data may be included for context.

+

BreedBase provides tools to generate layout files directly from trial metadata. These files can be exported and then uploaded to Field Book for data collection.

+

Walkthrough: Creating a Field Layout File in BreedBase

+
+
+
+

12.4 Importing Field Layout Files into Field Book

+

Field layout files can be imported into Field Book via cloud storage or directly from the device.

+
+

12.4.1 Option 1: Import from Cloud Storage

+

Requirements:

+
    +
  1. A generated field layout file.
    2. +
  2. Linked cloud service (e.g., Google Drive) on the device.
    3. +
  3. File uploaded to the cloud account.
    4. +
+

Walkthrough: Importing from Cloud Storage

+ +
+
+

12.4.2 Option 2: Import from Local Storage Device

+

Requirements:

+
    +
  1. A generated field layout file.
    2. +
  2. File transferred to the Android device’s local storage.
    3. +
+

Walkthrough: Importing from Local Device

+
+
+
+
+

12.5 Adding Traits to Field Book Using BreedBase

+

For exported data to be compatible with BreedBase, trait names in Field Book must match the exact names stored in the database.

+

BreedBase supports generation of trait files containing properly formatted trait names, types, units, and metadata.

+
+

12.5.1 Creating Trait Files in BreedBase

+

Walkthrough: Creating Trait Files in BreedBase

+ +
+
+

12.5.2 Importing Trait Files to Field Book

+

Walkthrough: Importing Trait Files

+ +
+
+
+
+

12.6 Collecting and Exporting Phenotype Data

+

This section describes the process for recording and exporting phenotype data using the Field Book application. It assumes that the required field layout and trait files have already been imported into the app.

+

Users will be guided through entering phenotypic observations in the field and exporting collected data in a format compatible with downstream processing and upload to BreedBase. This standardized workflow helps ensure data integrity, traceability, and seamless integration with breeding databases.

+
+

12.6.1 Collecting Phenotype Data in Field Book

+

Once field layout and trait files are loaded, data collection can begin. Field Book allows users to view each plot or plant entry, record trait values in structured forms, and automatically timestamp and geotag observations (if device settings allow).

+

Walkthrough: Collecting Data in Field Book

+
+
+

12.6.2 Exporting Phenotype Data from Field Book and Importing to BreedBase

+

After data collection is complete, Field Book allows users to export data as a .csv file. This file can then be uploaded to BreedBase for storage and analysis.

+

Walkthrough: Exporting Phenotype Data from Field Book

+

Once exported, the .csv file can be imported into BreedBase through the trial’s data upload interface. Proceed with uploading phenotype observations as described in the Managing Phenotypic Data section.

+

It is important that:

+
    +
  • Trait names match those defined in the database.
    • +
  • Plot or plant identifiers align with the original layout.
    • +
+
+
+
+
+

12.7 Using BrAPI for Data Transfer Between Field Book and BreedBase

+

Field Book supports integration with BreedBase via the BrAPI (Breeding API) specification. BrAPI enables direct exchange of trial metadata and phenotype observations between Field Book and BreedBase without the need for manual file uploads.

+

This walkthrough covers:

+
    +
  • Connecting Field Book to a BrAPI-enabled BreedBase server
    • +
  • Importing trial layout and traits from BreedBase to Field Book
    • +
  • Exporting collected phenotype data from Field Book to BreedBase
    • +
+

Prerequisites:

+
    +
  • You must have a BreedBase user account with permissions to access trial and trait data.
    • +
  • Your BreedBase instance must be configured with working BrAPI endpoints.
    • +
  • The Field Book app must be installed on an Android device with internet access.
    • +
+

Advantages of Using BrAPI:

+
    +
  • Reduces manual errors from file handling.
    • +
  • Supports real-time syncing of data.
    • +
  • Enables remote team coordination and efficient workflows.
    • +
+
+

12.7.1 Connecting Field Book to a BrAPI Server

+

Walkthrough: Connecting Field Book to a BreedBase BrAPI Server

+ +
+ +
+

12.7.3 Collecting Data

+

Proceed with entering phenotype observations as described in the Collecting Phenotype Data in Field Book section.

+

Ensure that:

+
    +
  • Trait values match expected formats (e.g., numeric, categorical).
    • +
  • Observations are saved regularly during field use.
    • +
+
+
+

12.7.4 Exporting Phenotype Data to BreedBase via BrAPI

+

Walkthrough: Sending Phenotype Data with BrAPI

+ +
+
+

12.7.5 Verifying Data in BreedBase

+

After exporting data to BreedBase via BrAPI, it is important to verify that the export was successful directly in BreedBase.

+
    +
  1. Log in to your BreedBase instance.
    2. +
  2. Navigate to the Manage Trials or Trial Detail page for the uploaded study.
    3. +
  3. Confirm that: +
      +
    • Trait data appears in the Phenotype Data tab.
      • +
    • Data points align with expected plot/plant identifiers.
      • +
    • Trait units and scales match those defined in the database.
      • +
    4. +
+
+
+

12.7.6 Troubleshooting

+
    +
  • Connection fails: Ensure you have the correct BrAPI server URL and valid credentials.
    • +
  • Traits do not load: Confirm that the trait exists in BreedBase.
    • +
  • Upload fails: Check for internet connectivity, correct user permissions, and trait formatting errors.
    • +
+ +
+
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/assets/images/Breedbase_HighRes.png b/docs/assets/images/Breedbase_HighRes.png new file mode 100644 index 0000000000..327f8a40aa Binary files /dev/null and b/docs/assets/images/Breedbase_HighRes.png differ diff --git a/docs/assets/images/FB_Section_Header.png b/docs/assets/images/FB_Section_Header.png new file mode 100644 index 0000000000..eb414f98f6 Binary files /dev/null and b/docs/assets/images/FB_Section_Header.png differ diff --git a/docs/assets/images/QR_FieldBook_PlayStore.png b/docs/assets/images/QR_FieldBook_PlayStore.png new file mode 100644 index 0000000000..f2f4ea058d Binary files /dev/null and b/docs/assets/images/QR_FieldBook_PlayStore.png differ diff --git a/docs/assets/images/ZoomIn.png b/docs/assets/images/ZoomIn.png new file mode 100644 index 0000000000..6f11f40ede Binary files /dev/null and b/docs/assets/images/ZoomIn.png differ diff --git a/docs/assets/images/ZoomOut.png b/docs/assets/images/ZoomOut.png new file mode 100644 index 0000000000..1a213c28d4 Binary files /dev/null and b/docs/assets/images/ZoomOut.png differ diff --git a/docs/assets/images/accession_upload_using_email.png b/docs/assets/images/accession_upload_using_email.png new file mode 100644 index 0000000000..beb797c95d Binary files /dev/null and b/docs/assets/images/accession_upload_using_email.png differ diff --git a/docs/assets/images/add_management_factor_name_dialog.png b/docs/assets/images/add_management_factor_name_dialog.png index 2cbd3afc76..de31941a46 100644 Binary files a/docs/assets/images/add_management_factor_name_dialog.png and b/docs/assets/images/add_management_factor_name_dialog.png differ diff --git a/docs/assets/images/add_transplanting_date.png b/docs/assets/images/add_transplanting_date.png new file mode 100644 index 0000000000..6f352b79d8 Binary files /dev/null and b/docs/assets/images/add_transplanting_date.png differ diff --git a/docs/assets/images/breeding_programs_screenshot.png b/docs/assets/images/breeding_programs_screenshot.png new file mode 100644 index 0000000000..370b8e77a8 Binary files /dev/null and b/docs/assets/images/breeding_programs_screenshot.png differ diff --git a/docs/assets/images/create_transcriptomics_protocol.png b/docs/assets/images/create_transcriptomics_protocol.png new file mode 100644 index 0000000000..bfb0c98061 Binary files /dev/null and b/docs/assets/images/create_transcriptomics_protocol.png differ diff --git a/docs/assets/images/cursorLocation.png b/docs/assets/images/cursorLocation.png new file mode 100644 index 0000000000..1ddde75b7c Binary files /dev/null and b/docs/assets/images/cursorLocation.png differ diff --git a/docs/assets/images/data_matrix_example.png b/docs/assets/images/data_matrix_example.png new file mode 100644 index 0000000000..a6dd103b73 Binary files /dev/null and b/docs/assets/images/data_matrix_example.png differ diff --git a/docs/assets/images/data_matrix_headers.png b/docs/assets/images/data_matrix_headers.png new file mode 100644 index 0000000000..01673de8a0 Binary files /dev/null and b/docs/assets/images/data_matrix_headers.png differ diff --git a/docs/assets/images/download_transcriptomics.png b/docs/assets/images/download_transcriptomics.png new file mode 100644 index 0000000000..1fb6eb1421 Binary files /dev/null and b/docs/assets/images/download_transcriptomics.png differ diff --git a/docs/assets/images/field_map_download_link.png b/docs/assets/images/field_map_download_link.png index fc40869e9e..74b4d1f84b 100644 Binary files a/docs/assets/images/field_map_download_link.png and b/docs/assets/images/field_map_download_link.png differ diff --git a/docs/assets/images/fieldmap_trial_layout.png b/docs/assets/images/fieldmap_trial_layout.png index 9e7a06f945..7e9ee21cfd 100644 Binary files a/docs/assets/images/fieldmap_trial_layout.png and b/docs/assets/images/fieldmap_trial_layout.png differ diff --git a/docs/assets/images/fieldmap_view_plot_image.png b/docs/assets/images/fieldmap_view_plot_image.png index 128d5b15ba..97eebb3ea3 100644 Binary files a/docs/assets/images/fieldmap_view_plot_image.png and b/docs/assets/images/fieldmap_view_plot_image.png differ diff --git a/docs/assets/images/filename_verify_success.png b/docs/assets/images/filename_verify_success.png new file mode 100644 index 0000000000..001eff461c Binary files /dev/null and b/docs/assets/images/filename_verify_success.png differ diff --git a/docs/assets/images/heatmap_assayed_trait_view.png b/docs/assets/images/heatmap_assayed_trait_view.png index 9615578e2f..97b2ee8865 100644 Binary files a/docs/assets/images/heatmap_assayed_trait_view.png and b/docs/assets/images/heatmap_assayed_trait_view.png differ diff --git a/docs/assets/images/heatmap_spatial_correction_button.png b/docs/assets/images/heatmap_spatial_correction_button.png new file mode 100644 index 0000000000..adc85fd6aa Binary files /dev/null and b/docs/assets/images/heatmap_spatial_correction_button.png differ diff --git a/docs/assets/images/image141.png b/docs/assets/images/image141.png index 48a1600278..705f69139b 100755 Binary files a/docs/assets/images/image141.png and b/docs/assets/images/image141.png differ diff --git a/docs/assets/images/image173.png b/docs/assets/images/image173.png index 6c8f54fc83..6ae4bbeba3 100755 Binary files a/docs/assets/images/image173.png and b/docs/assets/images/image173.png differ diff --git a/docs/assets/images/image187.png b/docs/assets/images/image187.png index 65ce76179a..3f3a75cb86 100755 Binary files a/docs/assets/images/image187.png and b/docs/assets/images/image187.png differ diff --git a/docs/assets/images/image332.png b/docs/assets/images/image332.png index bde0b92f80..4d823cc22d 100644 Binary files a/docs/assets/images/image332.png and b/docs/assets/images/image332.png differ diff --git a/docs/assets/images/image348.png b/docs/assets/images/image348.png index 04153215ef..cc815e4e0e 100644 Binary files a/docs/assets/images/image348.png and b/docs/assets/images/image348.png differ diff --git a/docs/assets/images/image85.png b/docs/assets/images/image85.png index 7c1725d6ca..fcb0f9f4ad 100755 Binary files a/docs/assets/images/image85.png and b/docs/assets/images/image85.png differ diff --git a/docs/assets/images/image87.png b/docs/assets/images/image87.png index acdefe7e32..36b4206c93 100755 Binary files a/docs/assets/images/image87.png and b/docs/assets/images/image87.png differ diff --git a/docs/assets/images/image_upload_dialog.png b/docs/assets/images/image_upload_dialog.png new file mode 100644 index 0000000000..1175d8e723 Binary files /dev/null and b/docs/assets/images/image_upload_dialog.png differ diff --git a/docs/assets/images/location_map.png b/docs/assets/images/location_map.png new file mode 100644 index 0000000000..1f6283c606 Binary files /dev/null and b/docs/assets/images/location_map.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_1.png b/docs/assets/images/manage_trials_upload_trial_1.png index 76abb52c88..98636bec52 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_1.png and b/docs/assets/images/manage_trials_upload_trial_1.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_2.png b/docs/assets/images/manage_trials_upload_trial_2.png index 8a7ee850c0..06e95f323d 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_2.png and b/docs/assets/images/manage_trials_upload_trial_2.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_3.png b/docs/assets/images/manage_trials_upload_trial_3.png index b8c039eec3..a9d23c486d 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_3.png and b/docs/assets/images/manage_trials_upload_trial_3.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_4.png b/docs/assets/images/manage_trials_upload_trial_4.png index b467b0cdd3..d5ac03dec0 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_4.png and b/docs/assets/images/manage_trials_upload_trial_4.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_5.png b/docs/assets/images/manage_trials_upload_trial_5.png index e90967b47d..ef4d8bec05 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_5.png and b/docs/assets/images/manage_trials_upload_trial_5.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_7.png b/docs/assets/images/manage_trials_upload_trial_7.png new file mode 100644 index 0000000000..82f077e640 Binary files /dev/null and b/docs/assets/images/manage_trials_upload_trial_7.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_8.png b/docs/assets/images/manage_trials_upload_trial_8.png new file mode 100644 index 0000000000..8f9a36dfc0 Binary files /dev/null and b/docs/assets/images/manage_trials_upload_trial_8.png differ diff --git a/docs/assets/images/manage_trials_upload_trial_template.png b/docs/assets/images/manage_trials_upload_trial_template.png index ee13e228ad..70b95f6316 100644 Binary files a/docs/assets/images/manage_trials_upload_trial_template.png and b/docs/assets/images/manage_trials_upload_trial_template.png differ diff --git a/docs/assets/images/multiple_trial_upload_with_email.png b/docs/assets/images/multiple_trial_upload_with_email.png new file mode 100644 index 0000000000..df9571559a Binary files /dev/null and b/docs/assets/images/multiple_trial_upload_with_email.png differ diff --git a/docs/assets/images/new_program_screenshot.png b/docs/assets/images/new_program_screenshot.png new file mode 100644 index 0000000000..812582a64a Binary files /dev/null and b/docs/assets/images/new_program_screenshot.png differ diff --git a/docs/assets/images/replace_plot_accession_form.png b/docs/assets/images/replace_plot_accession_form.png new file mode 100644 index 0000000000..970cff2460 Binary files /dev/null and b/docs/assets/images/replace_plot_accession_form.png differ diff --git a/docs/assets/images/roles.png b/docs/assets/images/roles.png new file mode 100644 index 0000000000..1b98be0580 Binary files /dev/null and b/docs/assets/images/roles.png differ diff --git a/docs/assets/images/spatial_layout_usage_help.png b/docs/assets/images/spatial_layout_usage_help.png new file mode 100644 index 0000000000..141765b324 Binary files /dev/null and b/docs/assets/images/spatial_layout_usage_help.png differ diff --git a/docs/assets/images/spatial_layout_usage_help_1.png b/docs/assets/images/spatial_layout_usage_help_1.png new file mode 100644 index 0000000000..5d68f69978 Binary files /dev/null and b/docs/assets/images/spatial_layout_usage_help_1.png differ diff --git a/docs/assets/images/spatial_layout_usage_help_2.png b/docs/assets/images/spatial_layout_usage_help_2.png new file mode 100644 index 0000000000..3fce0919f0 Binary files /dev/null and b/docs/assets/images/spatial_layout_usage_help_2.png differ diff --git a/docs/assets/images/spatial_layout_usage_help_3.png b/docs/assets/images/spatial_layout_usage_help_3.png new file mode 100644 index 0000000000..0aa1b38d48 Binary files /dev/null and b/docs/assets/images/spatial_layout_usage_help_3.png differ diff --git a/docs/assets/images/stock_add_image.png b/docs/assets/images/stock_add_image.png new file mode 100644 index 0000000000..feb9dfa254 Binary files /dev/null and b/docs/assets/images/stock_add_image.png differ diff --git a/docs/assets/images/tool_compatibility_data_summary.png b/docs/assets/images/tool_compatibility_data_summary.png new file mode 100644 index 0000000000..c0a2d97ed2 Binary files /dev/null and b/docs/assets/images/tool_compatibility_data_summary.png differ diff --git a/docs/assets/images/tool_compatibility_details.png b/docs/assets/images/tool_compatibility_details.png new file mode 100644 index 0000000000..e4876d8a9c Binary files /dev/null and b/docs/assets/images/tool_compatibility_details.png differ diff --git a/docs/assets/images/transcript_details_example.png b/docs/assets/images/transcript_details_example.png new file mode 100644 index 0000000000..2ebe7b9024 Binary files /dev/null and b/docs/assets/images/transcript_details_example.png differ diff --git a/docs/assets/images/transcript_details_headers.png b/docs/assets/images/transcript_details_headers.png new file mode 100644 index 0000000000..0391301b8b Binary files /dev/null and b/docs/assets/images/transcript_details_headers.png differ diff --git a/docs/assets/images/transcriptomics_menu.png b/docs/assets/images/transcriptomics_menu.png new file mode 100644 index 0000000000..314b8094e2 Binary files /dev/null and b/docs/assets/images/transcriptomics_menu.png differ diff --git a/docs/assets/images/trial_create_form_1.png b/docs/assets/images/trial_create_form_1.png index 51bb1fbe14..16a465448b 100644 Binary files a/docs/assets/images/trial_create_form_1.png and b/docs/assets/images/trial_create_form_1.png differ diff --git a/docs/assets/images/trial_create_form_2.png b/docs/assets/images/trial_create_form_2.png index 1ab75e6740..6c0e80b973 100644 Binary files a/docs/assets/images/trial_create_form_2.png and b/docs/assets/images/trial_create_form_2.png differ diff --git a/docs/assets/images/trial_create_form_3.png b/docs/assets/images/trial_create_form_3.png index a0cab2a8c7..716f37e0c2 100644 Binary files a/docs/assets/images/trial_create_form_3.png and b/docs/assets/images/trial_create_form_3.png differ diff --git a/docs/assets/images/trial_create_form_4.png b/docs/assets/images/trial_create_form_4.png index 47ade5f351..7057b9ea7a 100644 Binary files a/docs/assets/images/trial_create_form_4.png and b/docs/assets/images/trial_create_form_4.png differ diff --git a/docs/assets/images/trial_create_form_5.png b/docs/assets/images/trial_create_form_5.png index 1f6bf8eb1c..c544c04144 100644 Binary files a/docs/assets/images/trial_create_form_5.png and b/docs/assets/images/trial_create_form_5.png differ diff --git a/docs/assets/images/trial_create_form_6.png b/docs/assets/images/trial_create_form_6.png new file mode 100644 index 0000000000..80b405e5ba Binary files /dev/null and b/docs/assets/images/trial_create_form_6.png differ diff --git a/docs/assets/images/trial_create_form_7_bottom.png b/docs/assets/images/trial_create_form_7_bottom.png new file mode 100644 index 0000000000..ce10a0f747 Binary files /dev/null and b/docs/assets/images/trial_create_form_7_bottom.png differ diff --git a/docs/assets/images/trial_create_form_7_middle.png b/docs/assets/images/trial_create_form_7_middle.png new file mode 100644 index 0000000000..34ba171db2 Binary files /dev/null and b/docs/assets/images/trial_create_form_7_middle.png differ diff --git a/docs/assets/images/trial_create_form_7_top.png b/docs/assets/images/trial_create_form_7_top.png new file mode 100644 index 0000000000..98ad9300ff Binary files /dev/null and b/docs/assets/images/trial_create_form_7_top.png differ diff --git a/docs/assets/images/trial_create_form_8.png b/docs/assets/images/trial_create_form_8.png new file mode 100644 index 0000000000..6583861ded Binary files /dev/null and b/docs/assets/images/trial_create_form_8.png differ diff --git a/docs/assets/images/trial_create_manage_trials.png b/docs/assets/images/trial_create_manage_trials.png index 0bdd8cbc3b..a02b812ae8 100644 Binary files a/docs/assets/images/trial_create_manage_trials.png and b/docs/assets/images/trial_create_manage_trials.png differ diff --git a/docs/assets/images/trial_detail_page_add_plant_entries.png b/docs/assets/images/trial_detail_page_add_plant_entries.png index 3d6f19dbc8..022eb51c1d 100644 Binary files a/docs/assets/images/trial_detail_page_add_plant_entries.png and b/docs/assets/images/trial_detail_page_add_plant_entries.png differ diff --git a/docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png b/docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png index 4b6546f596..96a5ac7d75 100644 Binary files a/docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png and b/docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png differ diff --git a/docs/assets/images/trial_detail_page_navigator_1.png b/docs/assets/images/trial_detail_page_navigator_1.png new file mode 100644 index 0000000000..0c86fb6b52 Binary files /dev/null and b/docs/assets/images/trial_detail_page_navigator_1.png differ diff --git a/docs/assets/images/trial_detail_page_navigator_2.png b/docs/assets/images/trial_detail_page_navigator_2.png new file mode 100644 index 0000000000..43f1f7b5bf Binary files /dev/null and b/docs/assets/images/trial_detail_page_navigator_2.png differ diff --git a/docs/assets/images/trial_detail_page_navigator_3.png b/docs/assets/images/trial_detail_page_navigator_3.png new file mode 100644 index 0000000000..f70afc2eea Binary files /dev/null and b/docs/assets/images/trial_detail_page_navigator_3.png differ diff --git a/docs/assets/images/trial_detail_page_start.png b/docs/assets/images/trial_detail_page_start.png index da6aae3c43..71ae4ddc57 100644 Binary files a/docs/assets/images/trial_detail_page_start.png and b/docs/assets/images/trial_detail_page_start.png differ diff --git a/docs/assets/images/trial_detail_page_suppress_phenotype.png b/docs/assets/images/trial_detail_page_suppress_phenotype.png index 05b4ddca79..c7604288ca 100644 Binary files a/docs/assets/images/trial_detail_page_suppress_phenotype.png and b/docs/assets/images/trial_detail_page_suppress_phenotype.png differ diff --git a/docs/assets/images/trial_detail_page_view_repetitive_measurements.png b/docs/assets/images/trial_detail_page_view_repetitive_measurements.png new file mode 100644 index 0000000000..903be8d86b Binary files /dev/null and b/docs/assets/images/trial_detail_page_view_repetitive_measurements.png differ diff --git a/docs/assets/images/update_trial_metadata.png b/docs/assets/images/update_trial_metadata.png new file mode 100644 index 0000000000..eb04c25b21 Binary files /dev/null and b/docs/assets/images/update_trial_metadata.png differ diff --git a/docs/assets/images/vectorButtons.png b/docs/assets/images/vectorButtons.png new file mode 100644 index 0000000000..e047305a31 Binary files /dev/null and b/docs/assets/images/vectorButtons.png differ diff --git a/docs/assets/images/vectorHighlight.png b/docs/assets/images/vectorHighlight.png new file mode 100644 index 0000000000..dc858c617e Binary files /dev/null and b/docs/assets/images/vectorHighlight.png differ diff --git a/docs/assets/images/vectorViewer.png b/docs/assets/images/vectorViewer.png new file mode 100644 index 0000000000..6e47081717 Binary files /dev/null and b/docs/assets/images/vectorViewer.png differ diff --git a/docs/assets/images/wizard_related_phenotypes_download.png b/docs/assets/images/wizard_related_phenotypes_download.png index 0829653ab4..c2ca9c3314 100644 Binary files a/docs/assets/images/wizard_related_phenotypes_download.png and b/docs/assets/images/wizard_related_phenotypes_download.png differ diff --git a/docs/assets/style.css b/docs/assets/style.css new file mode 100644 index 0000000000..046f3253f8 --- /dev/null +++ b/docs/assets/style.css @@ -0,0 +1,15 @@ +/*--- LOGO ---*/ + +.toc-logo { + width: 200px !important; + object-fit: contain; + margin: 0 auto; +} + +.toc-logo img { + max-width: 100%; +} + +.tab { + margin-left: 40px; +} \ No newline at end of file diff --git a/docs/basic-website-usage.html b/docs/basic-website-usage.html new file mode 100644 index 0000000000..b08d4ac069 --- /dev/null +++ b/docs/basic-website-usage.html @@ -0,0 +1,726 @@ + + + + + + + Chapter 1 Basic Website Usage | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 1 Basic Website Usage

+

Breedbase is usually hosted on the cloud and is entirely web-based, with only a browser required to access and use it. The recommended browser is Firefox. To use Breedbase, you or your project will usually need to have your own Breedbase instance. Information how you can obtain your own instance can be obtained from the Breedbase project (https://breedbase.org/).

+

Once an instance is set up, the site needs to be configured and some metadata uploaded before you can design, run and analyze trials:

+
    +

  • define user accounts and their access privileges.

    • +

  • add a trait ontology with the traits that you require.

    • +

  • add the locations that you use in your breeding program

    • +

  • add the foundational germplasm and the respective pedigree data

    • +

  • “historical” trial data can be uploaded as needed. Usually more recent trials are prioritized over older trials

    • +
+

In this chapter, we will cover how you can manage the user accounts and some basic website features such as lists and how to navigate the menus. The other topics are covered in subsequent chapters. Note that Breedbase instances are highly customizable, and that not every instance of Breedbase will have the same options in the same location. Refer to the site specific documentation, if any, for site specific information.

+
+

1.1 Creating a User Account

+
+

1.1.1 Verifying first that you do not already have an account

+

Before creating an account, please verify first that you don’t already have an account. You can use “Search” menu to check if you already registered as a user.

+

In the “Search” menu, selecting the “People” tab and search your name. If nothing is found, proceed with the instructions below. Otherwise, continue by clicking the “Login” button. If you have forgotten your password, you can retrieve it by clicking the “Forgot your password?” link on the login page.

+
+
+

1.1.2 Creating a user account

+

On the right of the toolbar, click on “Login”, which will take you to the login dialog. On the login dialog, click on the link “sign up for an account.” It will take you to the page below:

+

+

Filling in all of the information, then click “Create Account.”

+

After you submit the information, an email will be sent to the provided email address. Check your email and click on the link to activate your account.

+
+
+
+

1.2 Managing your Account

+
+

1.2.1 Login

+

To login, click the “Login” link in the toolbar on any page and enter your username and password.

+

If you have forgotten your password, you can retrieve it by clicking the “Forgot your password?” link on the login page.

+

+
+
+

1.2.2 Editing Account Settings

+

Account settings can be edited by clicking on the “my profile” link displayed as your user name, on the right of the toolbar. You must be logged in order to access and change account settings.

+

+

You can add personal information to your account using the “View or update personal information” link.

+

To change your password, username, or your contact email, click on “Update account information” link. You must provide your old password before you can make any changes.

+

+
+
+

1.2.3 Changing Your Account Status: From “User” to “Submitter”

+

After you create an account, your account has a “user” status. This account has limited privileges.

+

Accounts with “user” status are able to:

+
    +
  • Change personal information
    • +
  • Post comments on pages
    • +
  • Post to the forum
    • +
+

To upgrade your account status to “submitter,” contact the database curators using the “contact” link provided at the footer of each page. Submitter accounts can add data, such as new plots, accessions, phenotypic data and images.

+
+
+

1.2.4 Submitting Feedback on an SGN Database

+

We appreciate your feedback! Feel free to submit any questions or suggestions by using the “Feedback” link provided at the footer of each page.

+
+
+ +
+

1.4 Working with Lists

+

Lists are collections of identifiers that are stored in the database. Lists can be composed of accessions, plots, traits, locations, and trials. A given list can only contain one type of items, for example, all items have to be of type location in a list of locations. Lists are attached to the individual user’s account, and can only be created and seen by the user while logged in, however, lists can be made public for all other users to see. SGN databases make heavy use of lists in a number of tools on the website. For example, trials are created using lists of accessions.

+
+

1.4.1 Creating lists

+

Lists can be generated in various ways:

+

One way to create a list is by clicking on the “Lists” link located on the toolbar.

+

+

To create a new list, enter the name of your new list and then click on the “New List” button. The name of the list can be anything, but should be unique and should be something to help you easily identify the list.

+

+

If the list already exists, it will appear on the “Your Lists” dialog. To add items to your list, click on the “View” icon to open the “List Contents” section.

+

+

On the “List Contents” page, enter items that you want to add to the list, then click on “Add” button.

+

+

The page will be updated and will display your items in a table at the bottom of the dialog It is possible to sort the list if you need.

+

+

Select the type of items in your list. To verify that the items that you added to your list are already stored in the database and that you selected a correct type for the items, click on the “Validate” button.

+

+

If those items are already in the database, a message will indicate that “This list passed validation”

+

+

Note that a list cannot contain duplicate elements. If a duplicate item is entered, the list manager will inform the user that the element is already in the list and will not add it again.

+

Another easy way to create a list is to use the 2.1, which can be accessed from the Search menu.

+
+
+

1.4.2 Viewing and editing lists

+

Lists can be viewed and edited using the “Lists” link on the toolbar. Clicking on the link will open a dialog that displays all of your lists, as well as an option to create new lists.

+

+

This page shows all lists that have been created, including those created by using the Search Wizard. You can view and edit your lists by using “Actions” buttons.

+
    +

  1. Clicking on the “view” icon will open a new window called “List Contents” that allows you to change the list name, the type of the list, add new items, or delete existing items.

    2. +

  2. Clicking on the “delete” icon will delete your list. Caution: this action cannot be undone.

    3. +

  3. Clicking on the “download” icon will download the contents of your list to your computer.

    4. +

  4. Clicking on the “make public” icon will make your list available for other users to view and use your list.

    5. +
+

+
+
+
+

1.5 User Permissions

+

Breedbase accounts are assigned one or more of four different roles to determine the level of access they have within the database. The possible roles are User, Submitter, Sequencer, and Curator. Each role grants specific permissions, and careful management of them helps prevent data from being altered or deleted in error.

+

+

Accounts are also assigned Breeding Program role(s) to grant access to the specfic breeding program(s) they work with.

+
    +
  • The User role gives an account permission to view and download data throughout the database.
    • +
  • The Submitter role gives an account permission to design field experiments and to upload and edit data using the tools in the “Manage” section. In order to submit and manage breeding data within a given breeding program, a submitter also must have a matching Breeding Program role.
    • +
  • The Sequencer role gives an account permission to design genotyping experiments and submit plates to a genotyping service.
    • +
  • The Curator role gives an account permission to do all of the above, as well as to delete data within the database. The Curator role also enables the addition or deletion of roles for all database accounts in the ‘Manage User Roles’ tool.
    • +
+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/data-analysis-tools.html b/docs/data-analysis-tools.html new file mode 100644 index 0000000000..e04b5c5481 --- /dev/null +++ b/docs/data-analysis-tools.html @@ -0,0 +1,821 @@ + + + + + + + Chapter 27 Data Analysis Tools | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 27 Data Analysis Tools

+

SGN databases provides several tools for phenotype data analysis, marker-assisted selection, sequence and expression analyses, as well as ontology browser. These tools can be found in the “Analyze” menu.

+

+
+

27.1 Selection Index

+

To determine rankings of accessions based on more than one desirable trait, SGN databases provide a “Selection Index” tool that allows you to specify a weighting on each trait. To access the tool, clicking on “Selection Index” in the “Analyze” menu.

+

+

On the Selection Index page, selecting a trial that you want to analyze.

+

+

After you selected a trial, you can find traits that were assayed in that trial in the “Trait” box.

+

+

Selecting a trait that you want to include in the analysis will open a new dialogue showing the selected trait and a box that you can assign a “Weight” of that trait. After you are done, you can continue by selecting another trait by clicking on “Add another trait” link.

+

+

After you selected another trait, this page will automatically update information for you by showing all of the traits that you selected for the analysis.

+

+

You also have options to choose a reference accession, choose to include accessions with missing phenotypes, scaling values to a reference accession. After you complete your setting, clicking on “Calculate Rankings”

+

+

The Selection Index tool will generate rankings of accessions based on the information that you specified. You can copy the results to your system clipboard, convert the table data to CSV format, or print the data.

+

+

Clicking on “Raw Average” will display average values of the phenotypes of those ranked accessions.

+

+

Selection Index tool also allows you to save top ranked accessions directly to “Lists”. You can retrieve top ranked accessions by selecting a number or a percent.

+

+
+
+

27.2 Genomic Selection

+

The prediction of breeding values for a trait is a one step or two steps process, depending on what stage in your breeding cycle you are. The first step is to build a prediction model for a trait using a training population of clones with phenotype and genotype data. If you have yet to select parents for crossing for your first cycle of selection you can use the breeding values of the training population. If you are at later stages of your selection program, you need to do the second step which is applying the prediction model on your selection population. All clones in your training and selection populations must exist in the database.

+

To use the genomic selection tool, on cassavabase.org, select “Genomic Selection” from the “analyze” pull-down menu.

+

+

There are three ways to build a model for a trait.

+
+

27.2.1 Building a Model - Method 1:

+

One way to build a model is, using a trait name, to search for trials in which the trait was phenotyped and use a trial or a combination of trials to build a model for the trait. For example, if you search for “mosaic disease severity, you will get a list of trials you can use as training populations.

+

+

You will get a list of trials (as shown below) in which the trait of your interested was phenotyped. From the list, you can use a single trial as a training population or combine several trails to form a training population for the prediction model of the trait. Let”s say, you want to create a training population using individuals from trials “cassava ibadan 2001/02” and “cassava ibadan 02/03” and build a model for “cassava mosaic disease severity” using all clones from the training population.

+

+

Select the trials to combine (the same coloured), click ‘done selecting’, click the “combine trials and build model” button, and you will get a model and its output for the trait. On the model detail page, you can view the description of input data used in the model, output from the model and search interface for selection populations the model you can apply to predict their breeding values. The description of the input data for the model includes the number of phenotyped clones, and the number of markers, scatter and frequency distribution plots for the phenotype data, relationship between the phenotype data and GEBVs, population structure. The model output includes model parameters, heritability of the trait , prediction accuracy, GEBVs of the individuals from the training population and marker effects.

+

+

Expand each section to see detailed information.

+

If you expand the ‘Trait phenotype data’ section, you will find plots to explore the phenotype data used in the model. You can assess the phenotype data using a scatter and histogram plots and the descriptive statistics.

+

+

+

A regression line between observed phenotypes and GEBVs shows the relationship between the two.

+

+

You can also explore if there is any sub-clustering in the training population using PCA.

+

+

To check the model accuracy, a 10-fold cross-validation test, expand the ‘model accuracy’ section.

+

+

Marker effects are also available for download. To do so, expanad the ‘Marker Effects’ section and click the ‘Download all marker effects’ link and you will get a tab delimited output to save on your computer.

+

+

The breeding values of the individuals used in the training population are displayed graphically. Mousing over each data point displays the clone and its breeding value. To examine better, you can zoom in into the plot by selecting an area on the plot. You can download them also by following the “Download all GEBVs” link.

+

+
+

Estimating breeding values in a selection population

+

If you already have a selection population (in the database), from the same model page, you can apply the model to the selection population and estimate breeding values for all the clones in the population. You can search for a selection population of clones in the database using the search interface or you can make a custom list of clones using the list interface. If you click the “search for all relevant selection populations”, you will see all relevant selection populations for that model. However, this option takes long time decause of the large set of populations in the database and the filtering. Therefore, the fastest way is to search for each of your selection populations by name. If you are logged in to the website you will also see a list of your custom set of genotyped clones.

+

+

To apply the model to a selection population, simply click your population name or “Predict Now” and you will get the predicted breeding values. When you see a name of (or acronym]) of the trait, follow the link and you will see an interactive plot of the breeding values and a link to download the breeding values of your selection population.

+

+
+
+
+

27.2.2 Building a Model - Method 2

+

Another way to build a model is by selecting a trial, instead of selecting and searching for a specific trait. This approach is useful when you know a particular trial that is relevant to the environment you are targeting to breed material for. This method allows you to build models and predict genomic estimated breeding values (GEBVs) for several traits within a single trial at once. You can also calculate selection index for your clones when GEBVs are estimated for multiple traits.

+

To do this select the “Genomic Selection” link found under the “analyze” menu. This will take you to the same home page as used with Method 1. However, instead of entering information to search for in “Search for a trait”, click on “Use a trait as a trial population”. This will expand a new menu that will show all available trials.

+

+

+

+

To begin creating the model, select the existing trial that you would like to use. In this example I will be using the trial and trait data from “Cassava Ibadan 2002/03” trial. Clicking on a trial will take you to a page where you can find information such as number of markers and number of phenotypes clones.

+

+

In addition to the number of phenotype clones and number of markers, the main page for the trial selected also has information and graphs on phenotypic correlation for all of the traits. By moving your cursor over the graph you can read the different values for correlation between two traits. A key with all of the trait names of the acronyms used can be found in the tab below the graph.

+

+

Below the “Training population summary” there is a tab for “Traits”. Clicking on this tab will show all available traits for the specific trial. You can create a model by choosing one or multiple traits in the trial and clicking “Build Model”. In this example, the traits for “cassava bacterial blight severity” and “cassava mosaic disease severity” have been selected.

+

+

Clicking on “Build Model” will take you to a new page with the models outputs for the traits. Under the “Genomic Selection Model Output” tab you can view the model output and the model accuracy. Clicking on any of the traits will take you to a page with information about the model output on that individual trait within the trial. There you can view all of the trait information that was seen in more detail in Method 1.

+

+

You can apply the models to simultaneously predict GEBVs for respective traits in a selection population by clicking on “Predict Now” or the name of the selection population. You can also apply the models to any set of genotyped clones that you can create using the “lists” feature. For more information on lists, click here. Follow the link to the trait name to view and download the predicted GEBVs for the trait in a selection population.

+

+

To compare clones based on their performance on multiple traits, you can calculate selection indices using the form below. Choose from the pulldown menu the population with predicted GEBVs for the traits and assign relative weights for each trait. The relative weight of each trait must be between 0 - 1. 0 being of least weight and importance, not wanting to consider that particular trait in selecting a genotype and 1 being a trait that you give highest importance.

+

In this example we will be using the “Cassava Ibadan 2002/03” population and assigning values to each of the traits. Remember that there is a list of acronyms and trait names at the bottom of the page for reference. After entering whatever values you would like for each trait click on the “Calculate” button to generate results. This will create a list of the top 10 genotypes that most closely match the criteria that you entered. The list will be displayed right below the “selection index” tab. This information can also be downloaded onto your computer by clicking on the “Download selection indices” link underneath the listed genotypes and selection indices.

+

+
+
+

27.2.3 Building a Model - Method 3

+

In addition to creating a model by searching for pre-existing traits or by preexisting trial name, models can also be created by using your own list of clones. This creates a model by using or creating a training population.

+

The page to use the third Method for creating a population model is the same as for the other two models. Select “Genomic Selection” from under the “analyze” menu of the main toolbar. This will take you to the Genomic Selection homepage and show you all three available methods to create a model. To see and use Method 3 scroll down and click on the tab labeled “Create a Training Population”. This will open a set of tools that will allow you to use pre-existing lists or to create a new list.

+

+

Once the “Create a Training Population” tab is opened you have the option to use a pre-existing list or create new one. To learn how to create a list, click here. The “Make a new list of plots” link will take you directly to the Search Wizard that is usually used to create lists.

+

Please note: the only lists that can be used in Method 3 to create a model are lists of plots and trials. If the pre-existing list is not of plots or trials (for example, traits, or locations) it will not show up and cannot be used as a training population. When you create you use a list of trials, the trials data will be combined to create a training data set.

+

To use your custom list of plots or trials as a training population, select the list and click “Go”. This will take you to a detail page for the training population.

+

+

From here on you can build models and predict breeding values as described in Method 2.

+
+
+
+

27.3 Genome Browsing

+

There are two ways to evaluate genotype information within the browser, from an accession detail page or a trial detail page.

+
+

27.3.1 Browsing Genotype data by Accession

+

If you are interested in browsing genotype information for a single accession, for example ‘BAHKYEHEMAA’, navigate to the accession detail page.

+

+

Near the bottom of the detail page is a collapsible section called “Accession Jbrowse”.

+

+

This section will contain a link to the accession jbrowse page if the necessary genotype data is available. Clicking the link should take you to a page that looks like this, a which point you can browsre the genotype data in the form of a vcf track aligned to the latest build of the genome.

+

+
+
+

27.3.2 Browsing Genotype data by Trial

+

If you are interested in browsing genotype information for the accessions within a given trial, navigate to the trial detail page.

+

+

Halfway down the page is a collapsible section called “Trial Jbrowse”. This section will contain a link to the trial jbrowse page if the necessary genotype data for at least two accessions planted in the trial is available.

+

+

Clicking the link should take you to a page that looks like this, a which point you can browse the genotype data in the form of vcf tracks aligned to the latest build of the genome.

+

+
+
+
+

27.4 Principal Component Analysis (PCA)

+

Principal component analysis helps estimate and visualize if there is sub-grouping of individuals within a dataset based on a number of variables. Currently, you can use marker data to run PCA on datasets.

+

You can run PCA from multiple places on the website. To do PCA on

+
    +

  1. individuals from a trial, go to the trial detail page and find the PCA tool under the “Analysis tools” section.

    2. +

  2. individuals from a training population you used in a GS modeling, do your modeling and find the PCA tool in the model output page.

    3. +

  3. individuals in a training population and selection population you applied the training model, do your modeling, apply the model on the selection population and find the PCA tool on the selection population prediction output page.

    4. +

  4. individuals in a list of accessions you created, for example using the search wizard, go to the “Analyze” menu and select the “Population Structure”, select your list of individuals and run PCA.

    5. +

  5. individuals from multiple trials, create a list of the trials using the search wizard, go to the “Analyze” menu and select the “Population Structure”, select your list of trials and run PCA.

    6. +
+

+

With all the options, you will get a interactive plot of the two PCs (shown below) that explain the largest variance. Point the cursor at any data point and you will see the individual name with its corresponding PCs scores. By clicking the ‘Download all PCs’, you can also download the 10 PCs scores in the text format.

+
+
+

27.5 ANOVA

+

Currently, ANOVA is implemented for a single trial (single year and single location). You can do ANOVA for RCBD, CRD, Alpha and Augmented trial designs. ANOVA is done using linear mixed effects model, where the genotypes is fixed effect and the replications and blocks are random effects. Fixed effect significance level is computed using “lmer” from “lmeTest” R package.

+

You can do ANOVA from two places: trial detail and training population detail. In both cases, if the phenotype data was from the supported trial designs,

+

– Go to the ANOVA section down in the trial or training population page

+

– Select the trait of you want to perform ANOVA

+

– Click the “Run ANOVA” and wait for the result

+

+
+
+

27.6 Clustering (K-Means, Hierarchical)

+

The K-Means method allows you to partition a dataset into groups (K number). The hierarchical clustering, agglomerative, allows you to explore underlying similarity and visualize in a tree structure (dendrogram) the different levels of similarities (clusters) among samples. You can do clustering based on marker data, phenotype data and GEBVs. When you use phenotype data, first clone averages for each trait are calculated. Both methods use Euclidean distance as a measure of similarity. For the hierachical clustering, the complete-linkage (farthest neighbour) method is used to link up clusters.

+

There are three pathways to using this tool.

+
    +
  1. When you have data in the form of a list or dataset from the search wizard:
    2. +
+
    +

  1. – go to the “Analyze” menu and select the clustering option

    2. +

  2. – make sure you are logged in

    3. +

  3. – Select the relevant genotyping protocol, if you are clustering using genotype data

    4. +

  4. – select your list or dataset, click “Go”

    5. +

  5. – select clustering type

    6. +

  6. – select the data type to use

    7. +

  7. – If you are running K-Means clustering, provide the number of partitions (K). If left blank it will partition the data set into optimal numbers for the dataset.

    8. +

  8. – click the “Run Cluster” and wait for the analysis to finish or queue the request and wait for an email with the analysis result.

    9. +

  9. – You can download the outputs following the download links.

    10. +
+
    +
  1. From the trial detail page:
    2. +
+
    +

  1. – Go to the “Analysis Tools” section

    2. +

  2. – Follow steps D to G in (1)

    3. +
+
    +
  1. In the solGS pipeline:
    2. +
+
    +
  1. – Once you you are in a model output put page, you will see a section where you can do clustering in the same way as above (option 2).
    2. +
+

K-Means clustering:

+

+

Hierarchical clustering:

+

+
+
+

27.7 Genetic Gain

+

You can check for genetic gain by comparing the the GEBVs of a training and a selection population. You can do this in the solGS pipepline once you build a model and apply the model to predict the GEBVs of a selection population. Once at that stage, you will see a section “Check Genetic Gain”. Select a selection population to compare with the training population and click the “Check Genetic Gain” button. The genetic gain will be visualized in boxplots. You can download the boxplot(s) as well as the GEBVs data used for the plot(s).

+

+
+
+

27.8 Kinship and Inbreeding Coefficients

+

This tool allows you to estimate genetic relatedness between a pair of individuals (kinship), homozygousity across loci in an individual (inbreeding coefficient), and genetic similarity of an individual relative to the rest of the population (averge kinship).

+

There are three pathways to using this tool.

+

(1) When you have a list or dataset clones, created from the search wizard:

+
    +

  1. – go to the “Analyze” menu and select the kinship and inbreeding

    2. +

  2. – make sure you are logged in

    3. +

  3. – Select the genotypic protocol for the marker data

    4. +

  4. – select your list or dataset of clones, click “Go”

    5. +

  5. – click the “Run Kinship” and wait for the analysis to finish, depending on the data size this may take minutes. You can choose to submit the analysis and wait for an email notice to view the results or wait for it to complete.

    6. +

  6. – You can download the output following the download links.

    7. +
+

(2) From the trial detail page:

+
    +

  1. – Go to the “Analysis Tools” section

    2. +

  2. – Follow steps C to G in (1)

    3. +
+

(3) In the solGS pipeline:

+
    +
  1. – Once you you are in a model output put page, scroll down to the “Kinship and Inbreeding” section and run kinship.
    2. +
+

+
+
+

27.9 Creating Crossing Groups

+

If you calculate selection index based on GEBVs of multiple traits, and you want to select a certain proportion of the indexed individuals (e.g. top 10%, or bottom 10%) and then you want to partition the selected individuals into a number of groups based on their genotypes, you can use the k-means clustering method.

+

The procedure is:

+
    +

  1. predict GEBVs for 2 or more traits

    2. +

  2. In the models output page, calculate selection indices. Note the name of the selection index data.

    3. +

  3. Go to the clustering section,

    4. +
+

– select the selection index data,

+

– select “K-means”,

+

– select “Genotype”,

+

– in the K-numbers textbox, fill in the number of groups you want to create,

+

– in the selection proportion textbox, fill in the proportion of the indexed individuals you want to select, e.g. for the top 15 percent, 15. if you wish to select bottom performing, prefix the number with minus sign (e.g. -15)

+

– then run cluster and wait for the result.

+

+
+
+

27.10 Search Wizard Genomic Relationship Matrix (GRM) Download

+

The genomic relationship matrix (GRM) is useful for understanding underlying structure in your population. Breedbase can compute the GRM using rrBLUP. First, select accessions in the search wizard and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. The GRM can be returned in a matrix format (.tsv) which shows all pairwise relationships between the selected accessions and is useful for visualization; alternatively, the GRM can be returned in a three-column format (.tsv) which is useful for programs like ASReml outside of Breedbase. The GRM can also be returned as a simple correlation heatmap image (.pdf). The GRM can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox “compute from parents”; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field.

+

+
+
+

27.11 Search Wizard Genome Wide Association Study (GWAS)

+

Performing a genome wide association study (GWAS) can determine genotypic markers which are significantly correlated to phenotypic traits. Breedbase can compute GWAS using rrBLUP. First, select accessions and trait(s) in the search wizard, and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Several traits can be selected in the search wizard; if the traits are not to be treated as repeated measurements then select “no” in the select box and this will tell Breedbase to return GWAS results independently for the selected traits. If the selected traits are indeed all repeated measurements then select “yes” in the select box and Breedbase will return as single GWAS analysis across all the phenotypic records. Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. GWAS results can be returned in a tabular format (.tsv) where the -log10(p-values) for the selected traits are returned; alternatively, the GWAS results can be returned as Manhattan and QQ plots for the selected traits. The GWAS can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox “compute from parents”; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field.

+

The GWAS will filter the data by the input MAF and missing data filters provided. After filtering the data is imputed using an “EM” method in rrBLUP. The Kinship matrix (GRM) is computed from the imputed genotypic data and used in the GWAS model. The GWAS uses fixed effects for different field trials and replicates in the phenotypic data.

+

+

+

+
+
+

27.12 Spectral Analysis

+

Visible and near-infrared spectroscopy (vis-NIRS) can be related to reference phenotypes through statistical models to produce accurate phenotypic predictions for unobserved samples, increasing phenotyping throughput. This technique is commonly used for predicting traits such as total starch, protein, carotenoid, and water content in many plant breeding programs. Breedbase implements the R package waves to offer training, evaluation, storage, and use of vis-NIRS prediction models for a wide range of spectrometers and phenotypes.

+

+
+

27.12.1 Dataset selection

+

In order to initiate an analysis, the user must select one or more datasets using 2.1. A dataset in Breedbase can contain observationUnit-level (plot-, plant-, or sample-level) trial metadata and phenotypic data from one or more trials. After navigating to the “NIRS” webpage under the “Manage” tab in Breedbase, the user can initiate an analysis and select one of these datasets as input for model training. An optional test dataset can be selected in the second step of the workflow.

+

+

+
+
+

27.12.2 Cross-validation

+

Five cross-validation schemes that represent scenarios common in plant breeding are available for this analysis. These include CV1, CV2, CV0, and CV00 as outlined below and described in depth by Jarquín et al. (2017) as well as random and stratified random sampling with a 70% training and 30% validation split. For those schemes from Jarquín et al. (2017), specific input datasets must be chosen based on genotype and environment relatedness. Cross-validation choices: +* Random sampling (70% training / 30% validation) +* Stratified random sampling, stratified based on phenotype (70% training / 30% validation) +* CV1, untested lines in tested environments +* CV2, tested lines in tested environments +* CV0, tested lines in untested environments +* CV00, untested lines in untested environments

+

+
+
+

27.12.3 Preprocessing

+

Preprocessing, also known as pretreatment, is often used to increase the signal to noise ratio in vis-NIR datasets. The waves function DoPreprocessing() applies functions from the stats and prospectr packages for common spectral preprocessing methods with the following options: +* Raw data (default) +* First derivative +* Second derivative +* Gap segment derivative +* Standard normal variate (SNV; Barnes et al., 1989) +* Savitzky-Golay polynomial smoothing (Savitzky and Golay, 1964)

+

For more information on preprocessing methods and implementation, see the waves manual, available through CRAN: waves.pdf

+

+
+
+

27.12.4 Algorithms

+

Several algorithms are available for calibration model development in Breedbase via the waves package. The TrainSpectralModel() function in waves performs hyperparameter tuning as applicable using these algorithms in combination with cross validation and train functions from the package caret. Currently, only regression algorithms are available, but classification algorithms such as PLS-DA and SVM clasification are under development. +* Partial least squares regression (PLSR; Wold et al., 1982; Wold et al., 1984) is a popular method for spectral calibrations, as it can handle datasets with high levels of collinearity, reducing the dimensionality of these data into orthogonal latent variables (components) that are then related to the response variable through a linear model (reviewed in Wold et al., 2001). To avoid overfitting, the number of these components included in the final model must be tuned for each use case. The PLSR algorithm from the pls package is implemented by waves. +* Random Forest regression (RF; Ho, 1995) is a machine learning algorithm based on a series of decision trees. The number of trees and decisions at each junction are hyperparameters that must be tuned for each model. Another feature of this algorithm is the ability to extract variable importance measures from a fitted model (Breiman, 2001). In Breedbase, this option is made available through implementation of the RF algorithm from the package randomForest in the waves function TrainSpectralModel(). This function outputs both model performance statistics and a downloadable table of importance values for each wavelength. It is worth noting that this algorithm is computationally intensive, so the user should not be alarmed if results do not come right away. Breedbase will continue to work in the background and will display results when the analysis is finished. +* Support vector machine regression (SVM; Vapnik, 2000) is another useful algorithm for working with high-dimension datasets consisting of non-linear data, with applications in both classification and regression. The package waves implements SVM with both linear and radial basis function kernels using the kernlab package.

+
+
+

27.12.5 Output: common model summary statistics

+

After training, model performance statistics are both displayed on a results webpage and made available for download in .csv format. These statistics are calculated by the TrainSpectralModel() function in waves using the caret and spectacles packages. Reported statistics include: +* Tuned parameters depending on the model algoritm +* Best.n.comp, the best number of components to be included in a PLSR model +* Best.ntree, the best number of trees in an RF model +* Best.mtry, the best number of variables to include at every decision point in an RF model +* RMSECV, the root mean squared error of cross-validation +* R2cv, the coefficient of multiple determination of cross-validation for PLSR models +* RMSEP, the root mean squared error of prediction +* R2p, the squared Pearson’s correlation between predicted and observed test set values +* RPD, the ratio of standard deviation of observed test set values to RMSEP +* RPIQ, the ratio of performance to interquartile distance +* CCC, the concordance correlation coefficient +* Bias, the average difference between the predicted and observed values +* SEP, the standard error of prediction +* R2sp, the squared Spearman”s rank correlation between predicted and observed test set values

+
+
+

27.12.6 Export model for later use

+

Once a model has been trained, it can be stored for later use. This action calls the SaveModel() function from waves. Metadata regarding the training dataset and other parameters specified by the user upon training initialization are stored alongside the model object itself in the database.

+

+
+
+

27.12.7 Predict phenotypes from an exported model (routine use)

+

For phenotype predictions, users select a dataset and can then choose from models in the database that were trained using the same spectrometer type as the spectral data in the chosen dataset. Predicted phenotypes are stored as such in the database and are tagged with an ontology term specifying that they are predicted and not directly measured. Metadata regarding the model used for prediction is stored alongside the predicted value in the database. Predicted phenotypes can then be used as normal in other Breedbase analysis tools such as the Selection Index and GWAS.

+

+

+
+
+

27.12.8 FAQ

+

The Breedbase Spectral Analysis Tool does not allow for prediction models involving data from multiple spectrometer types at once.

+

References +* Barnes, R.J., M.S. Dhanoa, and S.J. Lister. 1989. Standard normal variate transformation and de-trending of near-infrared diffuse reflectance spectra. Appl. Spectrosc. 43(5): 772-777. doi: 10.1366/0003702894202201. +* Breiman, L. 2001. Random forests. Mach. Learn. 45: 5-32. doi: 10.1201/9780429469275-8. +* Ho, T.K. 1995. Random decision forests. Proc. Int. Conf. Doc. Anal. Recognition, ICDAR 1: 278-282. doi: 10.1109/ICDAR.1995.598994. +* Jarquín, D., C. Lemes da Silva, R.C. Gaynor, J. Poland, A. Fritz, et al. 2017. Increasing Genomic-Enabled Prediction Accuracy by Modeling Genotype x Environment Interactions in Kansas Wheat. Plant Genome 10(2): plantgenome2016.12.0130. doi: 10.3835/plantgenome2016.12.0130. +* Johnson, R.A., and D.W. Wichern. 2007. Applied Multivariate Statistical Analysis (6th Edition). +De Maesschalck, R., D. Jouan-Rimbaud, and D.L. Massart. 2000. The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. doi: 10.1016/S0169-7439(99)00047-7. +* Mahalanobis, P.C. 1936. On the generalized distance in statistics. Natl. Inst. Sci. India. +* Savitzky, A., and M.J.E. Golay. 1964. Smoothing and Differentiation of Data by Simplified Least Squares Procedures. Anal. Chem. 36(8): 1627-1639. doi: 10.1021/ac60214a047. +* Shrestha, R., L. Matteis, M. Skofic, A. Portugal, G. McLaren, et al. 2012. Bridging the phenotypic and genetic data useful for integrated breeding through a data annotation using the Crop Ontology developed by the crop communities of practice. Front. Physiol. 3 AUG(August): 1-10. doi: 10.3389/fphys.2012.00326. +* Vapnik, V.N. 2000. The Nature of Statistical Learning Theory. Springer New York, New York, NY. +* Wold, S., A. Ruhe, H. Wold, and W.J. Dunn, III. 1984. The Collinearity Problem in Linear Regression. The Partial Least Squares (PLS) Approach to Generalized Inverses. SIAM J. Sci. Stat. Comput. 5(3): 735-743. doi: 10.1137/0905052. +* Wold, S., M. Sjöström, and L. Eriksson. 2001. PLS-regression: a basic tool of chemometrics. Chemom. Intell. Lab. Syst. 58(2): 109-130. doi: 10.1016/S0169-7439(01)00155-1.

+
+
+
+

27.13 General Mixed Model Tool

+

The general mixed model tool is available at /tools/mixedmodels and a link is provided from the Analyze menu.

+

To use the mixed model tool, first create dataset using the Wizard containing the data that you would like to analyze.

+

Select the Mixed Model tool from the Analyze menu.

+

You are presented with a workflow. On the first step of the workflow, select the dataset that you wish to analyze, click on “Choose dataset” to continue.

+

The second part of the workflow presents you with the traits in the dataset; you can select one or more traits from the lists using the select buttons. If you selected one trait, a bargraph of the trait distribution will be shown. Click the “Next step” button to move to the next screen.

+

+

On the model build screen, all the factors are displayed that are contained within the dataset. The factors are presented as a list of blue buttons that can be dragged using the mouse to areas on the screen which build a mixed model equation. The areas correspond to fixed factors, random factors, and optionally to more complex factors, such as fixed factors with interaction and fixe factors with vriable slope/intersects. Drag the available factors to the corresponding area. To calculate BLUPs for germplasm, drag the germplasmName button to the “Random factors” area. To calculate BLUEs, drag it to the “Fixed factors” area. The factors need to have different levels contained within them, for example, if there is only one trial in the dataset, it cannot be used as one of the factors. Click on “Run analysis and got to next step” to run the mixed model and display the results.

+

The result view contains two tabs, one with the raw data, either BLUPS or BLUEs, and the other the adjusted means from the raw data.

+

The results can be stored in the database as an analysis, by clicking the button provided on the top of the data.

+
+
+

27.14 Genomic Prediction of Cross Performance (GPCP)

+

The GPCP tool is available at /tools/gcpc and a link is provided from the Analyze menu. +The GCPC tool implements genomic prediction with additive and directional dominance in the linear mixed model to predict for cross performance.

+

Before using the tool, first create a dataset using the Wizard containing the data that you would like to analyze. (The dataset should have genotyping_protocols). +Second, create Selection Indices for your traits using Selection Index in Analyze Menu.

+

To use the tool, Select the GPCP tool from the Analyze menu.

+

Then, select the dataset with genotyping_protocols that you wish to analyze, click on “Proceed to Factor Selection” to load available factors that can be included in the model.

+

Select the factors you wish to include in the model either as Fixed or Random. Click “None” for factors that you don”t want to include in the model. Note that the “germplasmName” is factored as Random by default.

+

The next step is to select the selection index for your traits on the dropdown menu.

+

Once you are through, click “Run GPCP” to run the model. The output will be presented in form of a table with “ID”, “Parent1”, “Parent2” and their cross prediction merit organized in descending order. +The results will also have sex information based on whether the dataset has plant sexes available in the database.

+
+
+

27.15 Tool Compatibility

+

The dataset definition enables one to predict whether the dataset can be used in various analysis tools.

+

Upon creating a dataset, the site will automatically predict its compatibility with the available analysis tools and report these values on the dataset details page.

+

+

In the table, each tool will report to the user which traits are available to be analyzed based on phenotype data, and if different types of analyses are available, these will also be reported to the user. Some tools may give a warning sign to indicate that this dataset is compatible, but with potentially low sample sizes. Hover over the warning symbol to get a readout of the reason for the warning.

+

Below the table, there is a button that enables the user to re-calculate tool compatibility. This can be useful if a dataset is created before phenotypes are uploaded to a trial, since phenotype data is used in determining dataset compatibility. Even if the page appears to hang, do not worry; the compatibility check will continue in the background, and you can check later.

+

+

Below the tool compatibilities, there is also a summary of the data encompassed by the dataset and the criteria used for determining tool compatibility. Those criteria are used in the following way: +- Correlation: A dataset can be used in a correlation analysis if there are many phenotype measurements for different traits made on the same accession. +- Population Structure (PCA): A genotype PCA can be run if there are many accessions all genotyped with the same protocol. A phenotype PCA can be run if many accessions all have measurements on many traits. +- Clustering: Like a PCA, clustering can be done in both phenotype and genotype modes. They have the same requirements as PCA. +- Kinship & Inbreeding: A dataset with many accessions genotyped with the same protocol can be used for kinship analyses. +- Stability: A dataset containing many accessions with the same trait measured across multiple locations can be used in stability analyses. +- Heritability: This requires one or more trials with the same trait measured on the same accession across those trial(s). +- Mixed Models: This requires sufficient accession numbers, trait measurements, and trial designs. +- GWAS: A dataset is compatible with GWAS if there are many accessions genotyped for the same genotyping protocol, and the genotyping protocol has enough markers to run a GWAS. In addition, each accession needs to be phenotyped for a trait. +- Boxplotter: There must be sufficient trait measurements to make a boxplot of the trait.

+

In addition to being on the dataset details page, tool compatibilities may be listed on the dataset selection screens for analysis tools. The compatibilities are non-blocking; you may always try using a dataset in an analysis even if there are warnings or if it is deemed non-compatible. As before, you can hover over the warning symbols to see why a dataset may not have statistical power. For analyses with multiple modes, such as clustering and PCA, you can also hover over the compatibliity checkmark to see what types (phenotype or genotype) the dataset is compatible with.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000000..17e1c781f4 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,476 @@ + + + + + + + Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+ +
+

Introduction

+

Welcome to the Breedbase manual!

+
+

Use the table of contents in the left sidebar to navigate to the topic of your choice.

+

At any time you can select specific text in the manual to highlight or annotate it using Hypothesis. Open the Hypothesis sidebar on the right to view existing annotations.

+

You may also use the widgets at the top of the screen to:

+

- collapse the sidebar
- search for a specfic topic
- change the font size, font type, or the site theme
- download the manual as a pdf

+

Manual as a pdf can be download here also. Download

+
+

This manual is intended for database users.
+If you are a developer looking for software implementation details, please visit the developer wiki instead: https://github.com/solgenomics/sgn/wiki

+ + +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/libs/anchor-sections-1.1.0/anchor-sections-hash.css b/docs/libs/anchor-sections-1.1.0/anchor-sections-hash.css new file mode 100644 index 0000000000..b563ec97ed --- /dev/null +++ b/docs/libs/anchor-sections-1.1.0/anchor-sections-hash.css @@ -0,0 +1,2 @@ +/* Styles for section anchors */ +a.anchor-section::before {content: '#';font-size: 80%;} diff --git a/docs/libs/anchor-sections-1.1.0/anchor-sections.css b/docs/libs/anchor-sections-1.1.0/anchor-sections.css new file mode 100644 index 0000000000..041905f8b5 --- /dev/null +++ b/docs/libs/anchor-sections-1.1.0/anchor-sections.css @@ -0,0 +1,4 @@ +/* Styles for section anchors */ +a.anchor-section {margin-left: 10px; visibility: hidden; color: inherit;} +.hasAnchor:hover a.anchor-section {visibility: visible;} +ul > li > .anchor-section {display: none;} diff --git a/docs/libs/anchor-sections-1.1.0/anchor-sections.js b/docs/libs/anchor-sections-1.1.0/anchor-sections.js new file mode 100644 index 0000000000..fee005d95b --- /dev/null +++ b/docs/libs/anchor-sections-1.1.0/anchor-sections.js @@ -0,0 +1,11 @@ +document.addEventListener('DOMContentLoaded', function () { + // If section divs is used, we need to put the anchor in the child header + const headers = document.querySelectorAll("div.hasAnchor.section[class*='level'] > :first-child") + + headers.forEach(function (x) { + // Add to the header node + if (!x.classList.contains('hasAnchor')) x.classList.add('hasAnchor') + // Remove from the section or div created by Pandoc + x.parentElement.classList.remove('hasAnchor') + }) +}) diff --git a/docs/libs/gitbook-2.6.7/css/fontawesome/fontawesome-webfont.ttf b/docs/libs/gitbook-2.6.7/css/fontawesome/fontawesome-webfont.ttf new file mode 100644 index 0000000000..35acda2fa1 Binary files /dev/null and b/docs/libs/gitbook-2.6.7/css/fontawesome/fontawesome-webfont.ttf differ diff --git a/docs/libs/gitbook-2.6.7/css/plugin-bookdown.css b/docs/libs/gitbook-2.6.7/css/plugin-bookdown.css new file mode 100644 index 0000000000..ab7c20eb34 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-bookdown.css @@ -0,0 +1,105 @@ +.book .book-header h1 { + padding-left: 20px; + padding-right: 20px; +} +.book .book-header.fixed { + position: fixed; + right: 0; + top: 0; + left: 0; + border-bottom: 1px solid rgba(0,0,0,.07); +} +span.search-highlight { + background-color: #ffff88; +} +@media (min-width: 600px) { + .book.with-summary .book-header.fixed { + left: 300px; + } +} +@media (max-width: 1240px) { + .book .book-body.fixed { + top: 50px; + } + .book .book-body.fixed .body-inner { + top: auto; + } +} +@media (max-width: 600px) { + .book.with-summary .book-header.fixed { + left: calc(100% - 60px); + min-width: 300px; + } + .book.with-summary .book-body { + transform: none; + left: calc(100% - 60px); + min-width: 300px; + } + .book .book-body.fixed { + top: 0; + } +} + +.book .book-body.fixed .body-inner { + top: 50px; +} +.book .book-body .page-wrapper .page-inner section.normal sub, .book .book-body .page-wrapper .page-inner section.normal sup { + font-size: 85%; +} + +@media print { + .book .book-summary, .book .book-body .book-header, .fa { + display: none !important; + } + .book .book-body.fixed { + left: 0px; + } + .book .book-body,.book .book-body .body-inner, .book.with-summary { + overflow: visible !important; + } +} +.kable_wrapper { + border-spacing: 20px 0; + border-collapse: separate; + border: none; + margin: auto; +} +.kable_wrapper > tbody > tr > td { + vertical-align: top; +} +.book .book-body .page-wrapper .page-inner section.normal table tr.header { + border-top-width: 2px; +} +.book .book-body .page-wrapper .page-inner section.normal table tr:last-child td { + border-bottom-width: 2px; +} +.book .book-body .page-wrapper .page-inner section.normal table td, .book .book-body .page-wrapper .page-inner section.normal table th { + border-left: none; + border-right: none; +} +.book .book-body .page-wrapper .page-inner section.normal table.kable_wrapper > tbody > tr, .book .book-body .page-wrapper .page-inner section.normal table.kable_wrapper > tbody > tr > td { + border-top: none; +} +.book .book-body .page-wrapper .page-inner section.normal table.kable_wrapper > tbody > tr:last-child > td { + border-bottom: none; +} + +div.theorem, div.lemma, div.corollary, div.proposition, div.conjecture { + font-style: italic; +} +span.theorem, span.lemma, span.corollary, span.proposition, span.conjecture { + font-style: normal; +} +div.proof>*:last-child:after { + content: "\25a2"; + float: right; +} +.header-section-number { + padding-right: .5em; +} +#header .multi-author { + margin: 0.5em 0 -0.5em 0; +} +#header .date { + margin-top: 1.5em; +} diff --git a/docs/libs/gitbook-2.6.7/css/plugin-clipboard.css b/docs/libs/gitbook-2.6.7/css/plugin-clipboard.css new file mode 100644 index 0000000000..6844a70aa5 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-clipboard.css @@ -0,0 +1,18 @@ +div.sourceCode { + position: relative; +} + +.copy-to-clipboard-button { + position: absolute; + right: 0; + top: 0; + visibility: hidden; +} + +.copy-to-clipboard-button:focus { + outline: 0; +} + +div.sourceCode:hover > .copy-to-clipboard-button { + visibility: visible; +} diff --git a/docs/libs/gitbook-2.6.7/css/plugin-fontsettings.css b/docs/libs/gitbook-2.6.7/css/plugin-fontsettings.css new file mode 100644 index 0000000000..3fa6f35b2a --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-fontsettings.css @@ -0,0 +1,303 @@ +/* + * Theme 1 + */ +.color-theme-1 .dropdown-menu { + background-color: #111111; + border-color: #7e888b; +} +.color-theme-1 .dropdown-menu .dropdown-caret .caret-inner { + border-bottom: 9px solid #111111; +} +.color-theme-1 .dropdown-menu .buttons { + border-color: #7e888b; +} +.color-theme-1 .dropdown-menu .button { + color: #afa790; +} +.color-theme-1 .dropdown-menu .button:hover { + color: #73553c; +} +/* + * Theme 2 + */ +.color-theme-2 .dropdown-menu { + background-color: #2d3143; + border-color: #272a3a; +} +.color-theme-2 .dropdown-menu .dropdown-caret .caret-inner { + border-bottom: 9px solid #2d3143; +} +.color-theme-2 .dropdown-menu .buttons { + border-color: #272a3a; +} +.color-theme-2 .dropdown-menu .button { + color: #62677f; +} +.color-theme-2 .dropdown-menu .button:hover { + color: #f4f4f5; +} +.book .book-header .font-settings .font-enlarge { + line-height: 30px; + font-size: 1.4em; +} +.book .book-header .font-settings .font-reduce { + line-height: 30px; + font-size: 1em; +} + +/* sidebar transition background */ +div.book.color-theme-1 { + background: #f3eacb; +} +.book.color-theme-1 .book-body { + color: #704214; + background: #f3eacb; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section { + background: #f3eacb; +} + +/* sidebar transition background */ +div.book.color-theme-2 { + background: #1c1f2b; +} + +.book.color-theme-2 .book-body { + color: #bdcadb; + background: #1c1f2b; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section { + background: #1c1f2b; +} +.book.font-size-0 .book-body .page-inner section { + font-size: 1.2rem; +} +.book.font-size-1 .book-body .page-inner section { + font-size: 1.4rem; +} +.book.font-size-2 .book-body .page-inner section { + font-size: 1.6rem; +} +.book.font-size-3 .book-body .page-inner section { + font-size: 2.2rem; +} +.book.font-size-4 .book-body .page-inner section { + font-size: 4rem; +} +.book.font-family-0 { + font-family: Georgia, serif; +} +.book.font-family-1 { + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal { + color: #704214; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal a { + color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h1, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h2, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h3, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h4, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h5, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h6 { + color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h1, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h2 { + border-color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal h6 { + color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal hr { + background-color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal blockquote { + border-color: #c4b29f; + opacity: 0.9; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code { + background: #fdf6e3; + color: #657b83; + border-color: #f8df9c; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal .highlight { + background-color: inherit; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal table th, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal table td { + border-color: #f5d06c; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal table tr { + color: inherit; + background-color: #fdf6e3; + border-color: #444444; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal table tr:nth-child(2n) { + background-color: #fbeecb; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal { + color: #bdcadb; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal a { + color: #3eb1d0; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h1, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h2, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h3, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h4, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h5, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h6 { + color: #fffffa; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h1, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h2 { + border-color: #373b4e; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal h6 { + color: #373b4e; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal hr { + background-color: #373b4e; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal blockquote { + border-color: #373b4e; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code { + color: #9dbed8; + background: #2d3143; + border-color: #2d3143; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal .highlight { + background-color: #282a39; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal table th, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal table td { + border-color: #3b3f54; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal table tr { + color: #b6c2d2; + background-color: #2d3143; + border-color: #3b3f54; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal table tr:nth-child(2n) { + background-color: #35394b; +} +.book.color-theme-1 .book-header { + color: #afa790; + background: transparent; +} +.book.color-theme-1 .book-header .btn { + color: #afa790; +} +.book.color-theme-1 .book-header .btn:hover { + color: #73553c; + background: none; +} +.book.color-theme-1 .book-header h1 { + color: #704214; +} +.book.color-theme-2 .book-header { + color: #7e888b; + background: transparent; +} +.book.color-theme-2 .book-header .btn { + color: #3b3f54; +} +.book.color-theme-2 .book-header .btn:hover { + color: #fffff5; + background: none; +} +.book.color-theme-2 .book-header h1 { + color: #bdcadb; +} +.book.color-theme-1 .book-body .navigation { + color: #afa790; +} +.book.color-theme-1 .book-body .navigation:hover { + color: #73553c; +} +.book.color-theme-2 .book-body .navigation { + color: #383f52; +} +.book.color-theme-2 .book-body .navigation:hover { + color: #fffff5; +} +/* + * Theme 1 + */ +.book.color-theme-1 .book-summary { + color: #afa790; + background: #111111; + border-right: 1px solid rgba(0, 0, 0, 0.07); +} +.book.color-theme-1 .book-summary .book-search { + background: transparent; +} +.book.color-theme-1 .book-summary .book-search input, +.book.color-theme-1 .book-summary .book-search input:focus { + border: 1px solid transparent; +} +.book.color-theme-1 .book-summary ul.summary li.divider { + background: #7e888b; + box-shadow: none; +} +.book.color-theme-1 .book-summary ul.summary li i.fa-check { + color: #33cc33; +} +.book.color-theme-1 .book-summary ul.summary li.done > a { + color: #877f6a; +} +.book.color-theme-1 .book-summary ul.summary li a, +.book.color-theme-1 .book-summary ul.summary li span { + color: #877f6a; + background: transparent; + font-weight: normal; +} +.book.color-theme-1 .book-summary ul.summary li.active > a, +.book.color-theme-1 .book-summary ul.summary li a:hover { + color: #704214; + background: transparent; + font-weight: normal; +} +/* + * Theme 2 + */ +.book.color-theme-2 .book-summary { + color: #bcc1d2; + background: #2d3143; + border-right: none; +} +.book.color-theme-2 .book-summary .book-search { + background: transparent; +} +.book.color-theme-2 .book-summary .book-search input, +.book.color-theme-2 .book-summary .book-search input:focus { + border: 1px solid transparent; +} +.book.color-theme-2 .book-summary ul.summary li.divider { + background: #272a3a; + box-shadow: none; +} +.book.color-theme-2 .book-summary ul.summary li i.fa-check { + color: #33cc33; +} +.book.color-theme-2 .book-summary ul.summary li.done > a { + color: #62687f; +} +.book.color-theme-2 .book-summary ul.summary li a, +.book.color-theme-2 .book-summary ul.summary li span { + color: #c1c6d7; + background: transparent; + font-weight: 600; +} +.book.color-theme-2 .book-summary ul.summary li.active > a, +.book.color-theme-2 .book-summary ul.summary li a:hover { + color: #f4f4f5; + background: #252737; + font-weight: 600; +} diff --git a/docs/libs/gitbook-2.6.7/css/plugin-highlight.css b/docs/libs/gitbook-2.6.7/css/plugin-highlight.css new file mode 100644 index 0000000000..2aabd3debc --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-highlight.css @@ -0,0 +1,426 @@ +.book .book-body .page-wrapper .page-inner section.normal pre, +.book .book-body .page-wrapper .page-inner section.normal code { + /* http://jmblog.github.com/color-themes-for-google-code-highlightjs */ + /* Tomorrow Comment */ + /* Tomorrow Red */ + /* Tomorrow Orange */ + /* Tomorrow Yellow */ + /* Tomorrow Green */ + /* Tomorrow Aqua */ + /* Tomorrow Blue */ + /* Tomorrow Purple */ +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-comment, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-comment, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-title { + color: #8e908c; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-variable, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-variable, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-attribute, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-attribute, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-tag, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-tag, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-regexp, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-regexp, +.book .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-constant, +.book .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-constant, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-tag .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .xml .hljs-tag .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-pi, +.book .book-body .page-wrapper .page-inner section.normal code .xml .hljs-pi, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-doctype, +.book .book-body .page-wrapper .page-inner section.normal code .xml .hljs-doctype, +.book .book-body .page-wrapper .page-inner section.normal pre .html .hljs-doctype, +.book .book-body .page-wrapper .page-inner section.normal code .html .hljs-doctype, +.book .book-body .page-wrapper .page-inner section.normal pre .css .hljs-id, +.book .book-body .page-wrapper .page-inner section.normal code .css .hljs-id, +.book .book-body .page-wrapper .page-inner section.normal pre .css .hljs-class, +.book .book-body .page-wrapper .page-inner section.normal code .css .hljs-class, +.book .book-body .page-wrapper .page-inner section.normal pre .css .hljs-pseudo, +.book .book-body .page-wrapper .page-inner section.normal code .css .hljs-pseudo { + color: #c82829; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-number, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-number, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-preprocessor, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-preprocessor, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-pragma, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-pragma, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-built_in, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-built_in, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-literal, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-literal, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-params, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-params, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-constant, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-constant { + color: #f5871f; +} +.book .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-class .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-class .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal pre .css .hljs-rules .hljs-attribute, +.book .book-body .page-wrapper .page-inner section.normal code .css .hljs-rules .hljs-attribute { + color: #eab700; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-string, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-string, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-value, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-value, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-inheritance, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-inheritance, +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-header, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-header, +.book .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-symbol, +.book .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-symbol, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-cdata, +.book .book-body .page-wrapper .page-inner section.normal code .xml .hljs-cdata { + color: #718c00; +} +.book .book-body .page-wrapper .page-inner section.normal pre .css .hljs-hexcolor, +.book .book-body .page-wrapper .page-inner section.normal code .css .hljs-hexcolor { + color: #3e999f; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-function, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-function, +.book .book-body .page-wrapper .page-inner section.normal pre .python .hljs-decorator, +.book .book-body .page-wrapper .page-inner section.normal code .python .hljs-decorator, +.book .book-body .page-wrapper .page-inner section.normal pre .python .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .python .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-function .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-function .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-title .hljs-keyword, +.book .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-title .hljs-keyword, +.book .book-body .page-wrapper .page-inner section.normal pre .perl .hljs-sub, +.book .book-body .page-wrapper .page-inner section.normal code .perl .hljs-sub, +.book .book-body .page-wrapper .page-inner section.normal pre .javascript .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .javascript .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal pre .coffeescript .hljs-title, +.book .book-body .page-wrapper .page-inner section.normal code .coffeescript .hljs-title { + color: #4271ae; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs-keyword, +.book .book-body .page-wrapper .page-inner section.normal code .hljs-keyword, +.book .book-body .page-wrapper .page-inner section.normal pre .javascript .hljs-function, +.book .book-body .page-wrapper .page-inner section.normal code .javascript .hljs-function { + color: #8959a8; +} +.book .book-body .page-wrapper .page-inner section.normal pre .hljs, +.book .book-body .page-wrapper .page-inner section.normal code .hljs { + display: block; + background: white; + color: #4d4d4c; + padding: 0.5em; +} +.book .book-body .page-wrapper .page-inner section.normal pre .coffeescript .javascript, +.book .book-body .page-wrapper .page-inner section.normal code .coffeescript .javascript, +.book .book-body .page-wrapper .page-inner section.normal pre .javascript .xml, +.book .book-body .page-wrapper .page-inner section.normal code .javascript .xml, +.book .book-body .page-wrapper .page-inner section.normal pre .tex .hljs-formula, +.book .book-body .page-wrapper .page-inner section.normal code .tex .hljs-formula, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .javascript, +.book .book-body .page-wrapper .page-inner section.normal code .xml .javascript, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .vbscript, +.book .book-body .page-wrapper .page-inner section.normal code .xml .vbscript, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .css, +.book .book-body .page-wrapper .page-inner section.normal code .xml .css, +.book .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-cdata, +.book .book-body .page-wrapper .page-inner section.normal code .xml .hljs-cdata { + opacity: 0.5; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code { + /* + +Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull + +*/ + /* Solarized Green */ + /* Solarized Cyan */ + /* Solarized Blue */ + /* Solarized Yellow */ + /* Solarized Orange */ + /* Solarized Red */ + /* Solarized Violet */ +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs { + display: block; + padding: 0.5em; + background: #fdf6e3; + color: #657b83; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-comment, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-comment, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-template_comment, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-template_comment, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .diff .hljs-header, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .diff .hljs-header, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-doctype, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-doctype, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-pi, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-pi, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .lisp .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .lisp .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-javadoc, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-javadoc { + color: #93a1a1; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-keyword, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-keyword, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-winutils, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-winutils, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .method, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .method, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-addition, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-addition, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-tag, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .css .hljs-tag, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-request, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-request, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-status, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-status, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .nginx .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .nginx .hljs-title { + color: #859900; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-number, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-number, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-command, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-command, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-tag .hljs-value, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-tag .hljs-value, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-rules .hljs-value, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-rules .hljs-value, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-phpdoc, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-phpdoc, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .tex .hljs-formula, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .tex .hljs-formula, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-regexp, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-regexp, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-hexcolor, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-hexcolor, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-link_url, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-link_url { + color: #2aa198; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-localvars, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-localvars, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-chunk, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-chunk, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-decorator, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-decorator, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-built_in, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-built_in, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-identifier, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-identifier, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .vhdl .hljs-literal, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .vhdl .hljs-literal, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-id, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-id, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-function, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .css .hljs-function { + color: #268bd2; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-attribute, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-attribute, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-variable, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-variable, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .lisp .hljs-body, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .lisp .hljs-body, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .smalltalk .hljs-number, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .smalltalk .hljs-number, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-constant, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-constant, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-class .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-class .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-parent, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-parent, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .haskell .hljs-type, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .haskell .hljs-type, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-link_reference, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-link_reference { + color: #b58900; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-preprocessor, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-preprocessor, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-preprocessor .hljs-keyword, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-preprocessor .hljs-keyword, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-pragma, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-pragma, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-shebang, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-shebang, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-symbol, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-symbol, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-symbol .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-symbol .hljs-string, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .diff .hljs-change, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .diff .hljs-change, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-special, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-special, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-attr_selector, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-attr_selector, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-subst, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-subst, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-cdata, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-cdata, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .clojure .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .clojure .hljs-title, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-pseudo, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .css .hljs-pseudo, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-header, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-header { + color: #cb4b16; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-deletion, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-deletion, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-important, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-important { + color: #dc322f; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .hljs-link_label, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .hljs-link_label { + color: #6c71c4; +} +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal pre .tex .hljs-formula, +.book.color-theme-1 .book-body .page-wrapper .page-inner section.normal code .tex .hljs-formula { + background: #eee8d5; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code { + /* Tomorrow Night Bright Theme */ + /* Original theme - https://github.com/chriskempson/tomorrow-theme */ + /* http://jmblog.github.com/color-themes-for-google-code-highlightjs */ + /* Tomorrow Comment */ + /* Tomorrow Red */ + /* Tomorrow Orange */ + /* Tomorrow Yellow */ + /* Tomorrow Green */ + /* Tomorrow Aqua */ + /* Tomorrow Blue */ + /* Tomorrow Purple */ +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-comment, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-comment, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-title { + color: #969896; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-variable, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-variable, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-attribute, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-attribute, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-tag, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-tag, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-regexp, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-regexp, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-constant, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-constant, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-tag .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .hljs-tag .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-pi, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .hljs-pi, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-doctype, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .hljs-doctype, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .html .hljs-doctype, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .html .hljs-doctype, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-id, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .css .hljs-id, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-class, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .css .hljs-class, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-pseudo, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .css .hljs-pseudo { + color: #d54e53; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-number, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-number, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-preprocessor, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-preprocessor, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-pragma, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-pragma, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-built_in, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-built_in, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-literal, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-literal, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-params, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-params, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-constant, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-constant { + color: #e78c45; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-class .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-class .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-rules .hljs-attribute, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .css .hljs-rules .hljs-attribute { + color: #e7c547; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-string, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-string, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-value, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-value, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-inheritance, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-inheritance, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-header, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-header, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-symbol, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-symbol, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-cdata, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .hljs-cdata { + color: #b9ca4a; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .css .hljs-hexcolor, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .css .hljs-hexcolor { + color: #70c0b1; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-function, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-function, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .python .hljs-decorator, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .python .hljs-decorator, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .python .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .python .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-function .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-function .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .ruby .hljs-title .hljs-keyword, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .ruby .hljs-title .hljs-keyword, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .perl .hljs-sub, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .perl .hljs-sub, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .javascript .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .javascript .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .coffeescript .hljs-title, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .coffeescript .hljs-title { + color: #7aa6da; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs-keyword, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs-keyword, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .javascript .hljs-function, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .javascript .hljs-function { + color: #c397d8; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .hljs, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .hljs { + display: block; + background: black; + color: #eaeaea; + padding: 0.5em; +} +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .coffeescript .javascript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .coffeescript .javascript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .javascript .xml, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .javascript .xml, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .tex .hljs-formula, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .tex .hljs-formula, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .javascript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .javascript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .vbscript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .vbscript, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .css, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .css, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal pre .xml .hljs-cdata, +.book.color-theme-2 .book-body .page-wrapper .page-inner section.normal code .xml .hljs-cdata { + opacity: 0.5; +} diff --git a/docs/libs/gitbook-2.6.7/css/plugin-search.css b/docs/libs/gitbook-2.6.7/css/plugin-search.css new file mode 100644 index 0000000000..c85e557aaa --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-search.css @@ -0,0 +1,31 @@ +.book .book-summary .book-search { + padding: 6px; + background: transparent; + position: absolute; + top: -50px; + left: 0px; + right: 0px; + transition: top 0.5s ease; +} +.book .book-summary .book-search input, +.book .book-summary .book-search input:focus, +.book .book-summary .book-search input:hover { + width: 100%; + background: transparent; + border: 1px solid #ccc; + box-shadow: none; + outline: none; + line-height: 22px; + padding: 7px 4px; + color: inherit; + box-sizing: border-box; +} +.book.with-search .book-summary .book-search { + top: 0px; +} +.book.with-search .book-summary ul.summary { + top: 50px; +} +.with-search .summary li[data-level] a[href*=".html#"] { + display: none; +} diff --git a/docs/libs/gitbook-2.6.7/css/plugin-table.css b/docs/libs/gitbook-2.6.7/css/plugin-table.css new file mode 100644 index 0000000000..7fba1b9fb6 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/plugin-table.css @@ -0,0 +1 @@ +.book .book-body .page-wrapper .page-inner section.normal table{display:table;width:100%;border-collapse:collapse;border-spacing:0;overflow:auto}.book .book-body .page-wrapper .page-inner section.normal table td,.book .book-body .page-wrapper .page-inner section.normal table th{padding:6px 13px;border:1px solid #ddd}.book .book-body .page-wrapper .page-inner section.normal table tr{background-color:#fff;border-top:1px solid #ccc}.book .book-body .page-wrapper .page-inner section.normal table tr:nth-child(2n){background-color:#f8f8f8}.book .book-body .page-wrapper .page-inner section.normal table th{font-weight:700} diff --git a/docs/libs/gitbook-2.6.7/css/style.css b/docs/libs/gitbook-2.6.7/css/style.css new file mode 100644 index 0000000000..cba69b23b4 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/css/style.css @@ -0,0 +1,13 @@ +/*! normalize.css v2.1.0 | MIT License | git.io/normalize */img,legend{border:0}*{-webkit-font-smoothing:antialiased}sub,sup{position:relative}.book .book-body .page-wrapper .page-inner section.normal hr:after,.book-langs-index .inner .languages:after,.buttons:after,.dropdown-menu .buttons:after{clear:both}body,html{-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block}audio,canvas,video{display:inline-block}.hidden,[hidden]{display:none}audio:not([controls]){display:none;height:0}html{font-family:sans-serif}body,figure{margin:0}a:focus{outline:dotted thin}a:active,a:hover{outline:0}h1{font-size:2em;margin:.67em 0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}mark{background:#ff0;color:#000}code,kbd,pre,samp{font-family:monospace,serif;font-size:1em}pre{white-space:pre-wrap}q{quotes:"\201C" "\201D" "\2018" "\2019"}small{font-size:80%}sub,sup{font-size:75%;line-height:0;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}svg:not(:root){overflow:hidden}fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}legend{padding:0}button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}button,input{line-height:normal}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=search]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}input[type=search]::-webkit-search-cancel-button{margin-right:10px;}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}textarea{overflow:auto;vertical-align:top}table{border-collapse:collapse;border-spacing:0}/*! + * Preboot v2 + * + * Open sourced under MIT license by @mdo. + * Some variables and mixins from Bootstrap (Apache 2 license). + */.link-inherit,.link-inherit:focus,.link-inherit:hover{color:inherit}/*! + * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome + * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) + */@font-face{font-family:'FontAwesome';src:url('./fontawesome/fontawesome-webfont.ttf?v=4.7.0') format('truetype');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} +.book .book-header,.book .book-summary{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}.book-langs-index{width:100%;height:100%;padding:40px 0;margin:0;overflow:auto}@media (max-width:600px){.book-langs-index{padding:0}}.book-langs-index .inner{max-width:600px;width:100%;margin:0 auto;padding:30px;background:#fff;border-radius:3px}.book-langs-index .inner h3{margin:0}.book-langs-index .inner .languages{list-style:none;padding:20px 30px;margin-top:20px;border-top:1px solid #eee}.book-langs-index .inner .languages:after,.book-langs-index .inner .languages:before{content:" ";display:table;line-height:0}.book-langs-index .inner .languages li{width:50%;float:left;padding:10px 5px;font-size:16px}@media (max-width:600px){.book-langs-index .inner .languages li{width:100%;max-width:100%}}.book .book-header{overflow:visible;height:50px;padding:0 8px;z-index:2;font-size:.85em;color:#7e888b;background:0 0}.book .book-header .btn{display:block;height:50px;padding:0 15px;border-bottom:none;color:#ccc;text-transform:uppercase;line-height:50px;-webkit-box-shadow:none!important;box-shadow:none!important;position:relative;font-size:14px}.book .book-header .btn:hover{position:relative;text-decoration:none;color:#444;background:0 0}.book .book-header h1{margin:0;font-size:20px;font-weight:200;text-align:center;line-height:50px;opacity:0;padding-left:200px;padding-right:200px;-webkit-transition:opacity .2s ease;-moz-transition:opacity .2s ease;-o-transition:opacity .2s ease;transition:opacity .2s ease;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.book .book-header h1 a,.book .book-header h1 a:hover{color:inherit;text-decoration:none}@media (max-width:1000px){.book .book-header h1{display:none}}.book .book-header h1 i{display:none}.book .book-header:hover h1{opacity:1}.book.is-loading .book-header h1 i{display:inline-block}.book.is-loading .book-header h1 a{display:none}.dropdown{position:relative}.dropdown-menu{position:absolute;top:100%;left:0;z-index:100;display:none;float:left;min-width:160px;padding:0;margin:2px 0 0;list-style:none;font-size:14px;background-color:#fafafa;border:1px solid rgba(0,0,0,.07);border-radius:1px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175);background-clip:padding-box}.dropdown-menu.open{display:block}.dropdown-menu.dropdown-left{left:auto;right:4%}.dropdown-menu.dropdown-left .dropdown-caret{right:14px;left:auto}.dropdown-menu .dropdown-caret{position:absolute;top:-8px;left:14px;width:18px;height:10px;float:left;overflow:hidden}.dropdown-menu .dropdown-caret .caret-inner,.dropdown-menu .dropdown-caret .caret-outer{display:inline-block;top:0;border-left:9px solid transparent;border-right:9px solid transparent;position:absolute}.dropdown-menu .dropdown-caret .caret-outer{border-bottom:9px solid rgba(0,0,0,.1);height:auto;left:0;width:auto;margin-left:-1px}.dropdown-menu .dropdown-caret .caret-inner{margin-top:-1px;top:1px;border-bottom:9px solid #fafafa}.dropdown-menu .buttons{border-bottom:1px solid rgba(0,0,0,.07)}.dropdown-menu .buttons:after,.dropdown-menu .buttons:before{content:" ";display:table;line-height:0}.dropdown-menu .buttons:last-child{border-bottom:none}.dropdown-menu .buttons .button{border:0;background-color:transparent;color:#a6a6a6;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.alert,.dropdown-menu .buttons .button:hover{color:#444}.dropdown-menu .buttons .button:focus,.dropdown-menu .buttons .button:hover{outline:0}.dropdown-menu .buttons .button.size-2{width:50%}.dropdown-menu .buttons .button.size-3{width:33%}.alert{padding:15px;margin-bottom:20px;background:#eee;border-bottom:5px solid #ddd}.alert-success{background:#dff0d8;border-color:#d6e9c6;color:#3c763d}.alert-info{background:#d9edf7;border-color:#bce8f1;color:#31708f}.alert-danger{background:#f2dede;border-color:#ebccd1;color:#a94442}.alert-warning{background:#fcf8e3;border-color:#faebcc;color:#8a6d3b}.book .book-summary{position:absolute;top:0;left:-300px;bottom:0;z-index:1;width:300px;color:#364149;background:#fafafa;border-right:1px solid rgba(0,0,0,.07);-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-summary ul.summary{position:absolute;top:0;left:0;right:0;bottom:0;overflow-y:auto;list-style:none;margin:0;padding:0;-webkit-transition:top .5s ease;-moz-transition:top .5s ease;-o-transition:top .5s ease;transition:top .5s ease}.book .book-summary ul.summary li{list-style:none}.book .book-summary ul.summary li.divider{height:1px;margin:7px 0;overflow:hidden;background:rgba(0,0,0,.07)}.book .book-summary ul.summary li i.fa-check{display:none;position:absolute;right:9px;top:16px;font-size:9px;color:#3c3}.book .book-summary ul.summary li.done>a{color:#364149;font-weight:400}.book .book-summary ul.summary li.done>a i{display:inline}.book .book-summary ul.summary li a,.book .book-summary ul.summary li span{display:block;padding:10px 15px;border-bottom:none;color:#364149;background:0 0;text-overflow:ellipsis;overflow:hidden;white-space:nowrap;position:relative}.book .book-summary ul.summary li span{cursor:not-allowed;opacity:.3;filter:alpha(opacity=30)}.book .book-summary ul.summary li a:hover,.book .book-summary ul.summary li.active>a{color:#008cff;background:0 0;text-decoration:none}.book .book-summary ul.summary li ul{padding-left:20px}@media (max-width:600px){.book .book-summary{width:calc(100% - 60px);bottom:0;left:-100%}}.book.with-summary .book-summary{left:0}.book.without-animation .book-summary{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.book{position:relative;width:100%;height:100%}.book .book-body,.book .book-body .body-inner{position:absolute;top:0;left:0;overflow-y:auto;bottom:0;right:0}.book .book-body{color:#000;background:#fff;-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-body .page-wrapper{position:relative;outline:0}.book .book-body .page-wrapper .page-inner{max-width:800px;margin:0 auto;padding:20px 0 40px}.book .book-body .page-wrapper .page-inner section{margin:0;padding:5px 15px;background:#fff;border-radius:2px;line-height:1.7;font-size:1.6rem}.book .book-body .page-wrapper .page-inner .btn-group .btn{border-radius:0;background:#eee;border:0}@media (max-width:1240px){.book .book-body{-webkit-transition:-webkit-transform 250ms ease;-moz-transition:-moz-transform 250ms ease;-o-transition:-o-transform 250ms ease;transition:transform 250ms ease;padding-bottom:20px}.book .book-body .body-inner{position:static;min-height:calc(100% - 50px)}}@media (min-width:600px){.book.with-summary .book-body{left:300px}}@media (max-width:600px){.book.with-summary{overflow:hidden}.book.with-summary .book-body{-webkit-transform:translate(calc(100% - 60px),0);-moz-transform:translate(calc(100% - 60px),0);-ms-transform:translate(calc(100% - 60px),0);-o-transform:translate(calc(100% - 60px),0);transform:translate(calc(100% - 60px),0)}}.book.without-animation .book-body{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.buttons:after,.buttons:before{content:" ";display:table;line-height:0}.button{border:0;background:#eee;color:#666;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.button:hover{color:#444}.button:focus,.button:hover{outline:0}.button.size-2{width:50%}.button.size-3{width:33%}.book .book-body .page-wrapper .page-inner section{display:none}.book .book-body .page-wrapper .page-inner section.normal{display:block;word-wrap:break-word;overflow:hidden;color:#333;line-height:1.7;text-size-adjust:100%;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%}.book .book-body .page-wrapper .page-inner section.normal *{box-sizing:border-box;-webkit-box-sizing:border-box;}.book .book-body .page-wrapper .page-inner section.normal>:first-child{margin-top:0!important}.book .book-body .page-wrapper .page-inner section.normal>:last-child{margin-bottom:0!important}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal figure,.book .book-body .page-wrapper .page-inner section.normal img,.book .book-body .page-wrapper .page-inner section.normal pre,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal tr{page-break-inside:avoid}.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal p{orphans:3;widows:3}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5{page-break-after:avoid}.book .book-body .page-wrapper .page-inner section.normal b,.book .book-body .page-wrapper .page-inner section.normal strong{font-weight:700}.book .book-body .page-wrapper .page-inner section.normal em{font-style:italic}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal dl,.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal p,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal ul{margin-top:0;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal a{color:#4183c4;text-decoration:none;background:0 0}.book .book-body .page-wrapper .page-inner section.normal a:active,.book .book-body .page-wrapper .page-inner section.normal a:focus,.book .book-body .page-wrapper .page-inner section.normal a:hover{outline:0;text-decoration:underline}.book .book-body .page-wrapper .page-inner section.normal img{border:0;max-width:100%}.book .book-body .page-wrapper .page-inner section.normal hr{height:4px;padding:0;margin:1.7em 0;overflow:hidden;background-color:#e7e7e7;border:none}.book .book-body .page-wrapper .page-inner section.normal hr:after,.book .book-body .page-wrapper .page-inner section.normal hr:before{display:table;content:" "}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal h6{margin-top:1.275em;margin-bottom:.85em;}.book .book-body .page-wrapper .page-inner section.normal h1{font-size:2em}.book .book-body .page-wrapper .page-inner section.normal h2{font-size:1.75em}.book .book-body .page-wrapper .page-inner section.normal h3{font-size:1.5em}.book .book-body .page-wrapper .page-inner section.normal h4{font-size:1.25em}.book .book-body .page-wrapper .page-inner section.normal h5{font-size:1em}.book .book-body .page-wrapper .page-inner section.normal h6{font-size:1em;color:#777}.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal pre{font-family:Consolas,"Liberation Mono",Menlo,Courier,monospace;direction:ltr;border:none;color:inherit}.book .book-body .page-wrapper .page-inner section.normal pre{overflow:auto;word-wrap:normal;margin:0 0 1.275em;padding:.85em 1em;background:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal pre>code{display:inline;max-width:initial;padding:0;margin:0;overflow:initial;line-height:inherit;font-size:.85em;white-space:pre;background:0 0}.book .book-body .page-wrapper .page-inner section.normal pre>code:after,.book .book-body .page-wrapper .page-inner section.normal pre>code:before{content:normal}.book .book-body .page-wrapper .page-inner section.normal code{padding:.2em;margin:0;font-size:.85em;background-color:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal code:after,.book .book-body .page-wrapper .page-inner section.normal code:before{letter-spacing:-.2em;content:"\00a0"}.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal ul{padding:0 0 0 2em;margin:0 0 .85em}.book .book-body .page-wrapper .page-inner section.normal ol ol,.book .book-body .page-wrapper .page-inner section.normal ol ul,.book .book-body .page-wrapper .page-inner section.normal ul ol,.book .book-body .page-wrapper .page-inner section.normal ul ul{margin-top:0;margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal ol ol{list-style-type:lower-roman}.book .book-body .page-wrapper .page-inner section.normal blockquote{margin:0 0 .85em;padding:0 15px;opacity:0.75;border-left:4px solid #dcdcdc}.book .book-body .page-wrapper .page-inner section.normal blockquote:first-child{margin-top:0}.book .book-body .page-wrapper .page-inner section.normal blockquote:last-child{margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal dl{padding:0}.book .book-body .page-wrapper .page-inner section.normal dl dt{padding:0;margin-top:.85em;font-style:italic;font-weight:700}.book .book-body .page-wrapper .page-inner section.normal dl dd{padding:0 .85em;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal dd{margin-left:0}.book .book-body .page-wrapper .page-inner section.normal .glossary-term{cursor:help;text-decoration:underline}.book .book-body .navigation{position:absolute;top:50px;bottom:0;margin:0;max-width:150px;min-width:90px;display:flex;justify-content:center;align-content:center;flex-direction:column;font-size:40px;color:#ccc;text-align:center;-webkit-transition:all 350ms ease;-moz-transition:all 350ms ease;-o-transition:all 350ms ease;transition:all 350ms ease}.book .book-body .navigation:hover{text-decoration:none;color:#444}.book .book-body .navigation.navigation-next{right:0}.book .book-body .navigation.navigation-prev{left:0}@media (max-width:1240px){.book .book-body .navigation{position:static;top:auto;max-width:50%;width:50%;display:inline-block;float:left}.book .book-body .navigation.navigation-unique{max-width:100%;width:100%}}.book .book-body .page-wrapper .page-inner section.glossary{margin-bottom:40px}.book .book-body .page-wrapper .page-inner section.glossary h2 a,.book .book-body .page-wrapper .page-inner section.glossary h2 a:hover{color:inherit;text-decoration:none}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index{list-style:none;margin:0;padding:0}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index li{display:inline;margin:0 8px;white-space:nowrap}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-overflow-scrolling:auto;-webkit-tap-highlight-color:transparent;-webkit-text-size-adjust:none;-webkit-touch-callout:none}a{text-decoration:none}body,html{height:100%}html{font-size:62.5%}body{text-rendering:optimizeLegibility;font-smoothing:antialiased;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;letter-spacing:.2px;text-size-adjust:100%} +.book .book-summary ul.summary li a span {display:inline;padding:initial;overflow:visible;cursor:auto;opacity:1;} +/* show arrow before summary tag as in bootstrap */ +details > summary {display:list-item;cursor:pointer;} diff --git a/docs/libs/gitbook-2.6.7/js/app.min.js b/docs/libs/gitbook-2.6.7/js/app.min.js new file mode 100644 index 0000000000..643f1f9836 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/js/app.min.js @@ -0,0 +1 @@ +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o"'`]/g,reHasEscapedHtml=RegExp(reEscapedHtml.source),reHasUnescapedHtml=RegExp(reUnescapedHtml.source);var reEscape=/<%-([\s\S]+?)%>/g,reEvaluate=/<%([\s\S]+?)%>/g,reInterpolate=/<%=([\s\S]+?)%>/g;var reIsDeepProp=/\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\n\\]|\\.)*?\1)\]/,reIsPlainProp=/^\w*$/,rePropName=/[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\n\\]|\\.)*?)\2)\]/g;var reRegExpChars=/^[:!,]|[\\^$.*+?()[\]{}|\/]|(^[0-9a-fA-Fnrtuvx])|([\n\r\u2028\u2029])/g,reHasRegExpChars=RegExp(reRegExpChars.source);var reComboMark=/[\u0300-\u036f\ufe20-\ufe23]/g;var reEscapeChar=/\\(\\)?/g;var reEsTemplate=/\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g;var reFlags=/\w*$/;var reHasHexPrefix=/^0[xX]/;var reIsHostCtor=/^\[object .+?Constructor\]$/;var reIsUint=/^\d+$/;var reLatin1=/[\xc0-\xd6\xd8-\xde\xdf-\xf6\xf8-\xff]/g;var reNoMatch=/($^)/;var reUnescapedString=/['\n\r\u2028\u2029\\]/g;var reWords=function(){var upper="[A-Z\\xc0-\\xd6\\xd8-\\xde]",lower="[a-z\\xdf-\\xf6\\xf8-\\xff]+";return RegExp(upper+"+(?="+upper+lower+")|"+upper+"?"+lower+"|"+upper+"+|[0-9]+","g")}();var contextProps=["Array","ArrayBuffer","Date","Error","Float32Array","Float64Array","Function","Int8Array","Int16Array","Int32Array","Math","Number","Object","RegExp","Set","String","_","clearTimeout","isFinite","parseFloat","parseInt","setTimeout","TypeError","Uint8Array","Uint8ClampedArray","Uint16Array","Uint32Array","WeakMap"];var templateCounter=-1;var typedArrayTags={};typedArrayTags[float32Tag]=typedArrayTags[float64Tag]=typedArrayTags[int8Tag]=typedArrayTags[int16Tag]=typedArrayTags[int32Tag]=typedArrayTags[uint8Tag]=typedArrayTags[uint8ClampedTag]=typedArrayTags[uint16Tag]=typedArrayTags[uint32Tag]=true;typedArrayTags[argsTag]=typedArrayTags[arrayTag]=typedArrayTags[arrayBufferTag]=typedArrayTags[boolTag]=typedArrayTags[dateTag]=typedArrayTags[errorTag]=typedArrayTags[funcTag]=typedArrayTags[mapTag]=typedArrayTags[numberTag]=typedArrayTags[objectTag]=typedArrayTags[regexpTag]=typedArrayTags[setTag]=typedArrayTags[stringTag]=typedArrayTags[weakMapTag]=false;var cloneableTags={};cloneableTags[argsTag]=cloneableTags[arrayTag]=cloneableTags[arrayBufferTag]=cloneableTags[boolTag]=cloneableTags[dateTag]=cloneableTags[float32Tag]=cloneableTags[float64Tag]=cloneableTags[int8Tag]=cloneableTags[int16Tag]=cloneableTags[int32Tag]=cloneableTags[numberTag]=cloneableTags[objectTag]=cloneableTags[regexpTag]=cloneableTags[stringTag]=cloneableTags[uint8Tag]=cloneableTags[uint8ClampedTag]=cloneableTags[uint16Tag]=cloneableTags[uint32Tag]=true;cloneableTags[errorTag]=cloneableTags[funcTag]=cloneableTags[mapTag]=cloneableTags[setTag]=cloneableTags[weakMapTag]=false;var deburredLetters={"À":"A","Á":"A","Â":"A","Ã":"A","Ä":"A","Å":"A","à":"a","á":"a","â":"a","ã":"a","ä":"a","å":"a","Ç":"C","ç":"c","Ð":"D","ð":"d","È":"E","É":"E","Ê":"E","Ë":"E","è":"e","é":"e","ê":"e","ë":"e","Ì":"I","Í":"I","Î":"I","Ï":"I","ì":"i","í":"i","î":"i","ï":"i","Ñ":"N","ñ":"n","Ò":"O","Ó":"O","Ô":"O","Õ":"O","Ö":"O","Ø":"O","ò":"o","ó":"o","ô":"o","õ":"o","ö":"o","ø":"o","Ù":"U","Ú":"U","Û":"U","Ü":"U","ù":"u","ú":"u","û":"u","ü":"u","Ý":"Y","ý":"y","ÿ":"y","Æ":"Ae","æ":"ae","Þ":"Th","þ":"th","ß":"ss"};var htmlEscapes={"&":"&","<":"<",">":">",'"':""","'":"'","`":"`"};var htmlUnescapes={"&":"&","<":"<",">":">",""":'"',"'":"'","`":"`"};var objectTypes={function:true,object:true};var regexpEscapes={0:"x30",1:"x31",2:"x32",3:"x33",4:"x34",5:"x35",6:"x36",7:"x37",8:"x38",9:"x39",A:"x41",B:"x42",C:"x43",D:"x44",E:"x45",F:"x46",a:"x61",b:"x62",c:"x63",d:"x64",e:"x65",f:"x66",n:"x6e",r:"x72",t:"x74",u:"x75",v:"x76",x:"x78"};var stringEscapes={"\\":"\\","'":"'","\n":"n","\r":"r","\u2028":"u2028","\u2029":"u2029"};var freeExports=objectTypes[typeof exports]&&exports&&!exports.nodeType&&exports;var freeModule=objectTypes[typeof module]&&module&&!module.nodeType&&module;var freeGlobal=freeExports&&freeModule&&typeof global=="object"&&global&&global.Object&&global;var freeSelf=objectTypes[typeof self]&&self&&self.Object&&self;var freeWindow=objectTypes[typeof window]&&window&&window.Object&&window;var moduleExports=freeModule&&freeModule.exports===freeExports&&freeExports;var root=freeGlobal||freeWindow!==(this&&this.window)&&freeWindow||freeSelf||this;function baseCompareAscending(value,other){if(value!==other){var valIsNull=value===null,valIsUndef=value===undefined,valIsReflexive=value===value;var othIsNull=other===null,othIsUndef=other===undefined,othIsReflexive=other===other;if(value>other&&!othIsNull||!valIsReflexive||valIsNull&&!othIsUndef&&othIsReflexive||valIsUndef&&othIsReflexive){return 1}if(value-1){}return index}function charsRightIndex(string,chars){var index=string.length;while(index--&&chars.indexOf(string.charAt(index))>-1){}return index}function compareAscending(object,other){return baseCompareAscending(object.criteria,other.criteria)||object.index-other.index}function compareMultiple(object,other,orders){var index=-1,objCriteria=object.criteria,othCriteria=other.criteria,length=objCriteria.length,ordersLength=orders.length;while(++index=ordersLength){return result}var order=orders[index];return result*(order==="asc"||order===true?1:-1)}}return object.index-other.index}function deburrLetter(letter){return deburredLetters[letter]}function escapeHtmlChar(chr){return htmlEscapes[chr]}function escapeRegExpChar(chr,leadingChar,whitespaceChar){if(leadingChar){chr=regexpEscapes[chr]}else if(whitespaceChar){chr=stringEscapes[chr]}return"\\"+chr}function escapeStringChar(chr){return"\\"+stringEscapes[chr]}function indexOfNaN(array,fromIndex,fromRight){var length=array.length,index=fromIndex+(fromRight?0:-1);while(fromRight?index--:++index=9&&charCode<=13)||charCode==32||charCode==160||charCode==5760||charCode==6158||charCode>=8192&&(charCode<=8202||charCode==8232||charCode==8233||charCode==8239||charCode==8287||charCode==12288||charCode==65279)}function replaceHolders(array,placeholder){var index=-1,length=array.length,resIndex=-1,result=[];while(++index>>1;var MAX_SAFE_INTEGER=9007199254740991;var metaMap=WeakMap&&new WeakMap;var realNames={};function lodash(value){if(isObjectLike(value)&&!isArray(value)&&!(value instanceof LazyWrapper)){if(value instanceof LodashWrapper){return value}if(hasOwnProperty.call(value,"__chain__")&&hasOwnProperty.call(value,"__wrapped__")){return wrapperClone(value)}}return new LodashWrapper(value)}function baseLodash(){}function LodashWrapper(value,chainAll,actions){this.__wrapped__=value;this.__actions__=actions||[];this.__chain__=!!chainAll}var support=lodash.support={};lodash.templateSettings={escape:reEscape,evaluate:reEvaluate,interpolate:reInterpolate,variable:"",imports:{_:lodash}};function LazyWrapper(value){this.__wrapped__=value;this.__actions__=[];this.__dir__=1;this.__filtered__=false;this.__iteratees__=[];this.__takeCount__=POSITIVE_INFINITY;this.__views__=[]}function lazyClone(){var result=new LazyWrapper(this.__wrapped__);result.__actions__=arrayCopy(this.__actions__);result.__dir__=this.__dir__;result.__filtered__=this.__filtered__;result.__iteratees__=arrayCopy(this.__iteratees__);result.__takeCount__=this.__takeCount__;result.__views__=arrayCopy(this.__views__);return result}function lazyReverse(){if(this.__filtered__){var result=new LazyWrapper(this);result.__dir__=-1;result.__filtered__=true}else{result=this.clone();result.__dir__*=-1}return result}function lazyValue(){var array=this.__wrapped__.value(),dir=this.__dir__,isArr=isArray(array),isRight=dir<0,arrLength=isArr?array.length:0,view=getView(0,arrLength,this.__views__),start=view.start,end=view.end,length=end-start,index=isRight?end:start-1,iteratees=this.__iteratees__,iterLength=iteratees.length,resIndex=0,takeCount=nativeMin(length,this.__takeCount__);if(!isArr||arrLength=LARGE_ARRAY_SIZE?createCache(values):null,valuesLength=values.length;if(cache){indexOf=cacheIndexOf;isCommon=false;values=cache}outer:while(++indexlength?0:length+start}end=end===undefined||end>length?length:+end||0;if(end<0){end+=length}length=start>end?0:end>>>0;start>>>=0;while(startlength?0:length+start}end=end===undefined||end>length?length:+end||0;if(end<0){end+=length}length=start>end?0:end-start>>>0;start>>>=0;var result=Array(length);while(++index=LARGE_ARRAY_SIZE,seen=isLarge?createCache():null,result=[];if(seen){indexOf=cacheIndexOf;isCommon=false}else{isLarge=false;seen=iteratee?[]:result}outer:while(++index>>1,computed=array[mid];if((retHighest?computed<=value:computed2?sources[length-2]:undefined,guard=length>2?sources[2]:undefined,thisArg=length>1?sources[length-1]:undefined;if(typeof customizer=="function"){customizer=bindCallback(customizer,thisArg,5);length-=2}else{customizer=typeof thisArg=="function"?thisArg:undefined;length-=customizer?1:0}if(guard&&isIterateeCall(sources[0],sources[1],guard)){customizer=length<3?undefined:customizer;length=1}while(++index-1?collection[index]:undefined}return baseFind(collection,predicate,eachFunc)}}function createFindIndex(fromRight){return function(array,predicate,thisArg){if(!(array&&array.length)){return-1}predicate=getCallback(predicate,thisArg,3);return baseFindIndex(array,predicate,fromRight)}}function createFindKey(objectFunc){return function(object,predicate,thisArg){predicate=getCallback(predicate,thisArg,3);return baseFind(object,predicate,objectFunc,true)}}function createFlow(fromRight){return function(){var wrapper,length=arguments.length,index=fromRight?length:-1,leftIndex=0,funcs=Array(length);while(fromRight?index--:++index=LARGE_ARRAY_SIZE){return wrapper.plant(value).value()}var index=0,result=length?funcs[index].apply(this,args):value;while(++index=length||!nativeIsFinite(length)){return""}var padLength=length-strLength;chars=chars==null?" ":chars+"";return repeat(chars,nativeCeil(padLength/chars.length)).slice(0,padLength)}function createPartialWrapper(func,bitmask,thisArg,partials){var isBind=bitmask&BIND_FLAG,Ctor=createCtorWrapper(func);function wrapper(){var argsIndex=-1,argsLength=arguments.length,leftIndex=-1,leftLength=partials.length,args=Array(leftLength+argsLength);while(++leftIndexarrLength)){return false}while(++index-1&&value%1==0&&value-1&&value%1==0&&value<=MAX_SAFE_INTEGER}function isStrictComparable(value){return value===value&&!isObject(value)}function mergeData(data,source){var bitmask=data[1],srcBitmask=source[1],newBitmask=bitmask|srcBitmask,isCommon=newBitmask0){if(++count>=HOT_COUNT){return key}}else{count=0}return baseSetData(key,value)}}();function shimKeys(object){var props=keysIn(object),propsLength=props.length,length=propsLength&&object.length;var allowIndexes=!!length&&isLength(length)&&(isArray(object)||isArguments(object));var index=-1,result=[];while(++index=120?createCache(othIndex&&value):null}var array=arrays[0],index=-1,length=array?array.length:0,seen=caches[0];outer:while(++index-1){splice.call(array,fromIndex,1)}}return array}var pullAt=restParam(function(array,indexes){indexes=baseFlatten(indexes);var result=baseAt(array,indexes);basePullAt(array,indexes.sort(baseCompareAscending));return result});function remove(array,predicate,thisArg){var result=[];if(!(array&&array.length)){return result}var index=-1,indexes=[],length=array.length;predicate=getCallback(predicate,thisArg,3);while(++index2?arrays[length-2]:undefined,thisArg=length>1?arrays[length-1]:undefined;if(length>2&&typeof iteratee=="function"){length-=2}else{iteratee=length>1&&typeof thisArg=="function"?(--length,thisArg):undefined;thisArg=undefined}arrays.length=length;return unzipWith(arrays,iteratee,thisArg)});function chain(value){var result=lodash(value);result.__chain__=true;return result}function tap(value,interceptor,thisArg){interceptor.call(thisArg,value);return value}function thru(value,interceptor,thisArg){return interceptor.call(thisArg,value)}function wrapperChain(){return chain(this)}function wrapperCommit(){return new LodashWrapper(this.value(),this.__chain__)}var wrapperConcat=restParam(function(values){values=baseFlatten(values);return this.thru(function(array){return arrayConcat(isArray(array)?array:[toObject(array)],values)})});function wrapperPlant(value){var result,parent=this;while(parent instanceof baseLodash){var clone=wrapperClone(parent);if(result){previous.__wrapped__=clone}else{result=clone}var previous=clone;parent=parent.__wrapped__}previous.__wrapped__=value;return result}function wrapperReverse(){var value=this.__wrapped__;var interceptor=function(value){return wrapped&&wrapped.__dir__<0?value:value.reverse()};if(value instanceof LazyWrapper){var wrapped=value;if(this.__actions__.length){wrapped=new LazyWrapper(this)}wrapped=wrapped.reverse();wrapped.__actions__.push({func:thru,args:[interceptor],thisArg:undefined});return new LodashWrapper(wrapped,this.__chain__)}return this.thru(interceptor)}function wrapperToString(){return this.value()+""}function wrapperValue(){return baseWrapperValue(this.__wrapped__,this.__actions__)}var at=restParam(function(collection,props){return baseAt(collection,baseFlatten(props))});var countBy=createAggregator(function(result,value,key){hasOwnProperty.call(result,key)?++result[key]:result[key]=1});function every(collection,predicate,thisArg){var func=isArray(collection)?arrayEvery:baseEvery;if(thisArg&&isIterateeCall(collection,predicate,thisArg)){predicate=undefined}if(typeof predicate!="function"||thisArg!==undefined){predicate=getCallback(predicate,thisArg,3)}return func(collection,predicate)}function filter(collection,predicate,thisArg){var func=isArray(collection)?arrayFilter:baseFilter;predicate=getCallback(predicate,thisArg,3);return func(collection,predicate)}var find=createFind(baseEach);var findLast=createFind(baseEachRight,true);function findWhere(collection,source){return find(collection,baseMatches(source))}var forEach=createForEach(arrayEach,baseEach);var forEachRight=createForEach(arrayEachRight,baseEachRight);var groupBy=createAggregator(function(result,value,key){if(hasOwnProperty.call(result,key)){result[key].push(value)}else{result[key]=[value]}});function includes(collection,target,fromIndex,guard){var length=collection?getLength(collection):0;if(!isLength(length)){collection=values(collection);length=collection.length}if(typeof fromIndex!="number"||guard&&isIterateeCall(target,fromIndex,guard)){fromIndex=0}else{fromIndex=fromIndex<0?nativeMax(length+fromIndex,0):fromIndex||0}return typeof collection=="string"||!isArray(collection)&&isString(collection)?fromIndex<=length&&collection.indexOf(target,fromIndex)>-1:!!length&&getIndexOf(collection,target,fromIndex)>-1}var indexBy=createAggregator(function(result,value,key){result[key]=value});var invoke=restParam(function(collection,path,args){var index=-1,isFunc=typeof path=="function",isProp=isKey(path),result=isArrayLike(collection)?Array(collection.length):[];baseEach(collection,function(value){var func=isFunc?path:isProp&&value!=null?value[path]:undefined;result[++index]=func?func.apply(value,args):invokePath(value,path,args)});return result});function map(collection,iteratee,thisArg){var func=isArray(collection)?arrayMap:baseMap;iteratee=getCallback(iteratee,thisArg,3);return func(collection,iteratee)}var partition=createAggregator(function(result,value,key){result[key?0:1].push(value)},function(){return[[],[]]});function pluck(collection,path){return map(collection,property(path))}var reduce=createReduce(arrayReduce,baseEach);var reduceRight=createReduce(arrayReduceRight,baseEachRight);function reject(collection,predicate,thisArg){var func=isArray(collection)?arrayFilter:baseFilter;predicate=getCallback(predicate,thisArg,3);return func(collection,function(value,index,collection){return!predicate(value,index,collection)})}function sample(collection,n,guard){if(guard?isIterateeCall(collection,n,guard):n==null){collection=toIterable(collection);var length=collection.length;return length>0?collection[baseRandom(0,length-1)]:undefined}var index=-1,result=toArray(collection),length=result.length,lastIndex=length-1;n=nativeMin(n<0?0:+n||0,length);while(++index0){result=func.apply(this,arguments)}if(n<=1){func=undefined}return result}}var bind=restParam(function(func,thisArg,partials){var bitmask=BIND_FLAG;if(partials.length){var holders=replaceHolders(partials,bind.placeholder);bitmask|=PARTIAL_FLAG}return createWrapper(func,bitmask,thisArg,partials,holders)});var bindAll=restParam(function(object,methodNames){methodNames=methodNames.length?baseFlatten(methodNames):functions(object);var index=-1,length=methodNames.length;while(++indexwait){complete(trailingCall,maxTimeoutId)}else{timeoutId=setTimeout(delayed,remaining)}}function maxDelayed(){complete(trailing,timeoutId)}function debounced(){args=arguments;stamp=now();thisArg=this;trailingCall=trailing&&(timeoutId||!leading);if(maxWait===false){var leadingCall=leading&&!timeoutId}else{if(!maxTimeoutId&&!leading){lastCalled=stamp}var remaining=maxWait-(stamp-lastCalled),isCalled=remaining<=0||remaining>maxWait;if(isCalled){if(maxTimeoutId){maxTimeoutId=clearTimeout(maxTimeoutId)}lastCalled=stamp;result=func.apply(thisArg,args)}else if(!maxTimeoutId){maxTimeoutId=setTimeout(maxDelayed,remaining)}}if(isCalled&&timeoutId){timeoutId=clearTimeout(timeoutId)}else if(!timeoutId&&wait!==maxWait){timeoutId=setTimeout(delayed,wait)}if(leadingCall){isCalled=true;result=func.apply(thisArg,args)}if(isCalled&&!timeoutId&&!maxTimeoutId){args=thisArg=undefined}return result}debounced.cancel=cancel;return debounced}var defer=restParam(function(func,args){return baseDelay(func,1,args)});var delay=restParam(function(func,wait,args){return baseDelay(func,wait,args)});var flow=createFlow();var flowRight=createFlow(true);function memoize(func,resolver){if(typeof func!="function"||resolver&&typeof resolver!="function"){throw new TypeError(FUNC_ERROR_TEXT)}var memoized=function(){var args=arguments,key=resolver?resolver.apply(this,args):args[0],cache=memoized.cache;if(cache.has(key)){return cache.get(key)}var result=func.apply(this,args);memoized.cache=cache.set(key,result);return result};memoized.cache=new memoize.Cache;return memoized}var modArgs=restParam(function(func,transforms){transforms=baseFlatten(transforms);if(typeof func!="function"||!arrayEvery(transforms,baseIsFunction)){throw new TypeError(FUNC_ERROR_TEXT)}var length=transforms.length;return restParam(function(args){var index=nativeMin(args.length,length);while(index--){args[index]=transforms[index](args[index])}return func.apply(this,args)})});function negate(predicate){if(typeof predicate!="function"){throw new TypeError(FUNC_ERROR_TEXT)}return function(){return!predicate.apply(this,arguments)}}function once(func){return before(2,func)}var partial=createPartial(PARTIAL_FLAG);var partialRight=createPartial(PARTIAL_RIGHT_FLAG);var rearg=restParam(function(func,indexes){return createWrapper(func,REARG_FLAG,undefined,undefined,undefined,baseFlatten(indexes))});function restParam(func,start){if(typeof func!="function"){throw new TypeError(FUNC_ERROR_TEXT)}start=nativeMax(start===undefined?func.length-1:+start||0,0);return function(){var args=arguments,index=-1,length=nativeMax(args.length-start,0),rest=Array(length);while(++indexother}function gte(value,other){return value>=other}function isArguments(value){return isObjectLike(value)&&isArrayLike(value)&&hasOwnProperty.call(value,"callee")&&!propertyIsEnumerable.call(value,"callee")}var isArray=nativeIsArray||function(value){return isObjectLike(value)&&isLength(value.length)&&objToString.call(value)==arrayTag};function isBoolean(value){return value===true||value===false||isObjectLike(value)&&objToString.call(value)==boolTag}function isDate(value){return isObjectLike(value)&&objToString.call(value)==dateTag}function isElement(value){return!!value&&value.nodeType===1&&isObjectLike(value)&&!isPlainObject(value)}function isEmpty(value){if(value==null){return true}if(isArrayLike(value)&&(isArray(value)||isString(value)||isArguments(value)||isObjectLike(value)&&isFunction(value.splice))){return!value.length}return!keys(value).length}function isEqual(value,other,customizer,thisArg){customizer=typeof customizer=="function"?bindCallback(customizer,thisArg,3):undefined;var result=customizer?customizer(value,other):undefined;return result===undefined?baseIsEqual(value,other,customizer):!!result}function isError(value){return isObjectLike(value)&&typeof value.message=="string"&&objToString.call(value)==errorTag}function isFinite(value){return typeof value=="number"&&nativeIsFinite(value)}function isFunction(value){return isObject(value)&&objToString.call(value)==funcTag}function isObject(value){var type=typeof value;return!!value&&(type=="object"||type=="function")}function isMatch(object,source,customizer,thisArg){customizer=typeof customizer=="function"?bindCallback(customizer,thisArg,3):undefined;return baseIsMatch(object,getMatchData(source),customizer)}function isNaN(value){return isNumber(value)&&value!=+value}function isNative(value){if(value==null){return false}if(isFunction(value)){return reIsNative.test(fnToString.call(value))}return isObjectLike(value)&&reIsHostCtor.test(value)}function isNull(value){return value===null}function isNumber(value){return typeof value=="number"||isObjectLike(value)&&objToString.call(value)==numberTag}function isPlainObject(value){var Ctor;if(!(isObjectLike(value)&&objToString.call(value)==objectTag&&!isArguments(value))||!hasOwnProperty.call(value,"constructor")&&(Ctor=value.constructor,typeof Ctor=="function"&&!(Ctor instanceof Ctor))){return false}var result;baseForIn(value,function(subValue,key){result=key});return result===undefined||hasOwnProperty.call(value,result)}function isRegExp(value){return isObject(value)&&objToString.call(value)==regexpTag}function isString(value){return typeof value=="string"||isObjectLike(value)&&objToString.call(value)==stringTag}function isTypedArray(value){return isObjectLike(value)&&isLength(value.length)&&!!typedArrayTags[objToString.call(value)]}function isUndefined(value){return value===undefined}function lt(value,other){return value0;while(++index=nativeMin(start,end)&&value=0&&string.indexOf(target,position)==position}function escape(string){string=baseToString(string);return string&&reHasUnescapedHtml.test(string)?string.replace(reUnescapedHtml,escapeHtmlChar):string}function escapeRegExp(string){string=baseToString(string);return string&&reHasRegExpChars.test(string)?string.replace(reRegExpChars,escapeRegExpChar):string||"(?:)"}var kebabCase=createCompounder(function(result,word,index){return result+(index?"-":"")+word.toLowerCase()});function pad(string,length,chars){string=baseToString(string);length=+length;var strLength=string.length;if(strLength>=length||!nativeIsFinite(length)){return string}var mid=(length-strLength)/2,leftLength=nativeFloor(mid),rightLength=nativeCeil(mid);chars=createPadding("",rightLength,chars);return chars.slice(0,leftLength)+string+chars}var padLeft=createPadDir();var padRight=createPadDir(true);function parseInt(string,radix,guard){if(guard?isIterateeCall(string,radix,guard):radix==null){radix=0}else if(radix){radix=+radix}string=trim(string);return nativeParseInt(string,radix||(reHasHexPrefix.test(string)?16:10))}function repeat(string,n){var result="";string=baseToString(string);n=+n;if(n<1||!string||!nativeIsFinite(n)){return result}do{if(n%2){result+=string}n=nativeFloor(n/2);string+=string}while(n);return result}var snakeCase=createCompounder(function(result,word,index){return result+(index?"_":"")+word.toLowerCase()});var startCase=createCompounder(function(result,word,index){return result+(index?" ":"")+(word.charAt(0).toUpperCase()+word.slice(1))});function startsWith(string,target,position){string=baseToString(string);position=position==null?0:nativeMin(position<0?0:+position||0,string.length);return string.lastIndexOf(target,position)==position}function template(string,options,otherOptions){var settings=lodash.templateSettings;if(otherOptions&&isIterateeCall(string,options,otherOptions)){options=otherOptions=undefined}string=baseToString(string);options=assignWith(baseAssign({},otherOptions||options),settings,assignOwnDefaults);var imports=assignWith(baseAssign({},options.imports),settings.imports,assignOwnDefaults),importsKeys=keys(imports),importsValues=baseValues(imports,importsKeys);var isEscaping,isEvaluating,index=0,interpolate=options.interpolate||reNoMatch,source="__p += '";var reDelimiters=RegExp((options.escape||reNoMatch).source+"|"+interpolate.source+"|"+(interpolate===reInterpolate?reEsTemplate:reNoMatch).source+"|"+(options.evaluate||reNoMatch).source+"|$","g");var sourceURL="//# sourceURL="+("sourceURL"in options?options.sourceURL:"lodash.templateSources["+ ++templateCounter+"]")+"\n";string.replace(reDelimiters,function(match,escapeValue,interpolateValue,esTemplateValue,evaluateValue,offset){interpolateValue||(interpolateValue=esTemplateValue);source+=string.slice(index,offset).replace(reUnescapedString,escapeStringChar);if(escapeValue){isEscaping=true;source+="' +\n__e("+escapeValue+") +\n'"}if(evaluateValue){isEvaluating=true;source+="';\n"+evaluateValue+";\n__p += '"}if(interpolateValue){source+="' +\n((__t = ("+interpolateValue+")) == null ? '' : __t) +\n'"}index=offset+match.length;return match});source+="';\n";var variable=options.variable;if(!variable){source="with (obj) {\n"+source+"\n}\n"}source=(isEvaluating?source.replace(reEmptyStringLeading,""):source).replace(reEmptyStringMiddle,"$1").replace(reEmptyStringTrailing,"$1;");source="function("+(variable||"obj")+") {\n"+(variable?"":"obj || (obj = {});\n")+"var __t, __p = ''"+(isEscaping?", __e = _.escape":"")+(isEvaluating?", __j = Array.prototype.join;\n"+"function print() { __p += __j.call(arguments, '') }\n":";\n")+source+"return __p\n}";var result=attempt(function(){return Function(importsKeys,sourceURL+"return "+source).apply(undefined,importsValues)});result.source=source;if(isError(result)){throw result}return result}function trim(string,chars,guard){var value=string;string=baseToString(string);if(!string){return string}if(guard?isIterateeCall(value,chars,guard):chars==null){return string.slice(trimmedLeftIndex(string),trimmedRightIndex(string)+1)}chars=chars+"";return string.slice(charsLeftIndex(string,chars),charsRightIndex(string,chars)+1)}function trimLeft(string,chars,guard){var value=string;string=baseToString(string);if(!string){return string}if(guard?isIterateeCall(value,chars,guard):chars==null){return string.slice(trimmedLeftIndex(string))}return string.slice(charsLeftIndex(string,chars+""))}function trimRight(string,chars,guard){var value=string;string=baseToString(string);if(!string){return string}if(guard?isIterateeCall(value,chars,guard):chars==null){return string.slice(0,trimmedRightIndex(string)+1)}return string.slice(0,charsRightIndex(string,chars+"")+1)}function trunc(string,options,guard){if(guard&&isIterateeCall(string,options,guard)){options=undefined}var length=DEFAULT_TRUNC_LENGTH,omission=DEFAULT_TRUNC_OMISSION;if(options!=null){if(isObject(options)){var separator="separator"in options?options.separator:separator;length="length"in options?+options.length||0:length;omission="omission"in options?baseToString(options.omission):omission}else{length=+options||0}}string=baseToString(string);if(length>=string.length){return string}var end=length-omission.length;if(end<1){return omission}var result=string.slice(0,end);if(separator==null){return result+omission}if(isRegExp(separator)){if(string.slice(end).search(separator)){var match,newEnd,substring=string.slice(0,end);if(!separator.global){separator=RegExp(separator.source,(reFlags.exec(separator)||"")+"g")}separator.lastIndex=0;while(match=separator.exec(substring)){newEnd=match.index}result=result.slice(0,newEnd==null?end:newEnd)}}else if(string.indexOf(separator,end)!=end){var index=result.lastIndexOf(separator);if(index>-1){result=result.slice(0,index)}}return result+omission}function unescape(string){string=baseToString(string);return string&&reHasEscapedHtml.test(string)?string.replace(reEscapedHtml,unescapeHtmlChar):string}function words(string,pattern,guard){if(guard&&isIterateeCall(string,pattern,guard)){pattern=undefined}string=baseToString(string);return string.match(pattern||reWords)||[]}var attempt=restParam(function(func,args){try{return func.apply(undefined,args)}catch(e){return isError(e)?e:new Error(e)}});function callback(func,thisArg,guard){if(guard&&isIterateeCall(func,thisArg,guard)){thisArg=undefined}return isObjectLike(func)?matches(func):baseCallback(func,thisArg)}function constant(value){return function(){return value}}function identity(value){return value}function matches(source){return baseMatches(baseClone(source,true))}function matchesProperty(path,srcValue){return baseMatchesProperty(path,baseClone(srcValue,true))}var method=restParam(function(path,args){return function(object){return invokePath(object,path,args)}});var methodOf=restParam(function(object,args){return function(path){return invokePath(object,path,args)}});function mixin(object,source,options){if(options==null){var isObj=isObject(source),props=isObj?keys(source):undefined,methodNames=props&&props.length?baseFunctions(source,props):undefined;if(!(methodNames?methodNames.length:isObj)){methodNames=false;options=source;source=object;object=this}}if(!methodNames){methodNames=baseFunctions(source,keys(source))}var chain=true,index=-1,isFunc=isFunction(object),length=methodNames.length;if(options===false){chain=false}else if(isObject(options)&&"chain"in options){chain=options.chain}while(++index0||end<0)){return new LazyWrapper(result)}if(start<0){result=result.takeRight(-start)}else if(start){result=result.drop(start)}if(end!==undefined){end=+end||0;result=end<0?result.dropRight(-end):result.take(end-start)}return result};LazyWrapper.prototype.takeRightWhile=function(predicate,thisArg){return this.reverse().takeWhile(predicate,thisArg).reverse()};LazyWrapper.prototype.toArray=function(){return this.take(POSITIVE_INFINITY)};baseForOwn(LazyWrapper.prototype,function(func,methodName){var checkIteratee=/^(?:filter|map|reject)|While$/.test(methodName),retUnwrapped=/^(?:first|last)$/.test(methodName),lodashFunc=lodash[retUnwrapped?"take"+(methodName=="last"?"Right":""):methodName];if(!lodashFunc){return}lodash.prototype[methodName]=function(){var args=retUnwrapped?[1]:arguments,chainAll=this.__chain__,value=this.__wrapped__,isHybrid=!!this.__actions__.length,isLazy=value instanceof LazyWrapper,iteratee=args[0],useLazy=isLazy||isArray(value);if(useLazy&&checkIteratee&&typeof iteratee=="function"&&iteratee.length!=1){isLazy=useLazy=false}var interceptor=function(value){return retUnwrapped&&chainAll?lodashFunc(value,1)[0]:lodashFunc.apply(undefined,arrayPush([value],args))};var action={func:thru,args:[interceptor],thisArg:undefined},onlyLazy=isLazy&&!isHybrid;if(retUnwrapped&&!chainAll){if(onlyLazy){value=value.clone();value.__actions__.push(action);return func.call(value)}return lodashFunc.call(undefined,this.value())[0]}if(!retUnwrapped&&useLazy){value=onlyLazy?value:new LazyWrapper(this);var result=func.apply(value,args);result.__actions__.push(action);return new LodashWrapper(result,chainAll)}return this.thru(interceptor)}});arrayEach(["join","pop","push","replace","shift","sort","splice","split","unshift"],function(methodName){var func=(/^(?:replace|split)$/.test(methodName)?stringProto:arrayProto)[methodName],chainName=/^(?:push|sort|unshift)$/.test(methodName)?"tap":"thru",retUnwrapped=/^(?:join|pop|replace|shift)$/.test(methodName);lodash.prototype[methodName]=function(){var args=arguments;if(retUnwrapped&&!this.__chain__){return func.apply(this.value(),args)}return this[chainName](function(value){return func.apply(value,args)})}});baseForOwn(LazyWrapper.prototype,function(func,methodName){var lodashFunc=lodash[methodName];if(lodashFunc){var key=lodashFunc.name,names=realNames[key]||(realNames[key]=[]);names.push({name:methodName,func:lodashFunc})}});realNames[createHybridWrapper(undefined,BIND_KEY_FLAG).name]=[{name:"wrapper",func:undefined}];LazyWrapper.prototype.clone=lazyClone;LazyWrapper.prototype.reverse=lazyReverse;LazyWrapper.prototype.value=lazyValue;lodash.prototype.chain=wrapperChain;lodash.prototype.commit=wrapperCommit;lodash.prototype.concat=wrapperConcat;lodash.prototype.plant=wrapperPlant;lodash.prototype.reverse=wrapperReverse;lodash.prototype.toString=wrapperToString;lodash.prototype.run=lodash.prototype.toJSON=lodash.prototype.valueOf=lodash.prototype.value=wrapperValue;lodash.prototype.collect=lodash.prototype.map;lodash.prototype.head=lodash.prototype.first;lodash.prototype.select=lodash.prototype.filter;lodash.prototype.tail=lodash.prototype.rest;return lodash}var _=runInContext();if(typeof define=="function"&&typeof define.amd=="object"&&define.amd){root._=_;define(function(){return _})}else if(freeExports&&freeModule){if(moduleExports){(freeModule.exports=_)._=_}else{freeExports._=_}}else{root._=_}}).call(this)}).call(this,typeof global!=="undefined"?global:typeof self!=="undefined"?self:typeof window!=="undefined"?window:{})},{}],3:[function(require,module,exports){(function(window,document,undefined){var _MAP={8:"backspace",9:"tab",13:"enter",16:"shift",17:"ctrl",18:"alt",20:"capslock",27:"esc",32:"space",33:"pageup",34:"pagedown",35:"end",36:"home",37:"left",38:"up",39:"right",40:"down",45:"ins",46:"del",91:"meta",93:"meta",224:"meta"};var _KEYCODE_MAP={106:"*",107:"+",109:"-",110:".",111:"/",186:";",187:"=",188:",",189:"-",190:".",191:"/",192:"`",219:"[",220:"\\",221:"]",222:"'"};var _SHIFT_MAP={"~":"`","!":"1","@":"2","#":"3",$:"4","%":"5","^":"6","&":"7","*":"8","(":"9",")":"0",_:"-","+":"=",":":";",'"':"'","<":",",">":".","?":"/","|":"\\"};var _SPECIAL_ALIASES={option:"alt",command:"meta",return:"enter",escape:"esc",plus:"+",mod:/Mac|iPod|iPhone|iPad/.test(navigator.platform)?"meta":"ctrl"};var _REVERSE_MAP;for(var i=1;i<20;++i){_MAP[111+i]="f"+i}for(i=0;i<=9;++i){_MAP[i+96]=i}function _addEvent(object,type,callback){if(object.addEventListener){object.addEventListener(type,callback,false);return}object.attachEvent("on"+type,callback)}function _characterFromEvent(e){if(e.type=="keypress"){var character=String.fromCharCode(e.which);if(!e.shiftKey){character=character.toLowerCase()}return character}if(_MAP[e.which]){return _MAP[e.which]}if(_KEYCODE_MAP[e.which]){return _KEYCODE_MAP[e.which]}return String.fromCharCode(e.which).toLowerCase()}function _modifiersMatch(modifiers1,modifiers2){return modifiers1.sort().join(",")===modifiers2.sort().join(",")}function _eventModifiers(e){var modifiers=[];if(e.shiftKey){modifiers.push("shift")}if(e.altKey){modifiers.push("alt")}if(e.ctrlKey){modifiers.push("ctrl")}if(e.metaKey){modifiers.push("meta")}return modifiers}function _preventDefault(e){if(e.preventDefault){e.preventDefault();return}e.returnValue=false}function _stopPropagation(e){if(e.stopPropagation){e.stopPropagation();return}e.cancelBubble=true}function _isModifier(key){return key=="shift"||key=="ctrl"||key=="alt"||key=="meta"}function _getReverseMap(){if(!_REVERSE_MAP){_REVERSE_MAP={};for(var key in _MAP){if(key>95&&key<112){continue}if(_MAP.hasOwnProperty(key)){_REVERSE_MAP[_MAP[key]]=key}}}return _REVERSE_MAP}function _pickBestAction(key,modifiers,action){if(!action){action=_getReverseMap()[key]?"keydown":"keypress"}if(action=="keypress"&&modifiers.length){action="keydown"}return action}function _keysFromString(combination){if(combination==="+"){return["+"]}combination=combination.replace(/\+{2}/g,"+plus");return combination.split("+")}function _getKeyInfo(combination,action){var keys;var key;var i;var modifiers=[];keys=_keysFromString(combination);for(i=0;i1){_bindSequence(combination,sequence,callback,action);return}info=_getKeyInfo(combination,action);self._callbacks[info.key]=self._callbacks[info.key]||[];_getMatches(info.key,info.modifiers,{type:info.action},sequenceName,combination,level);self._callbacks[info.key][sequenceName?"unshift":"push"]({callback:callback,modifiers:info.modifiers,action:info.action,seq:sequenceName,level:level,combo:combination})}self._bindMultiple=function(combinations,callback,action){for(var i=0;i-1){return false}if(_belongsTo(element,self.target)){return false}return element.tagName=="INPUT"||element.tagName=="SELECT"||element.tagName=="TEXTAREA"||element.isContentEditable};Mousetrap.prototype.handleKey=function(){var self=this;return self._handleKey.apply(self,arguments)};Mousetrap.init=function(){var documentMousetrap=Mousetrap(document);for(var method in documentMousetrap){if(method.charAt(0)!=="_"){Mousetrap[method]=function(method){return function(){return documentMousetrap[method].apply(documentMousetrap,arguments)}}(method)}}};Mousetrap.init();window.Mousetrap=Mousetrap;if(typeof module!=="undefined"&&module.exports){module.exports=Mousetrap}if(typeof define==="function"&&define.amd){define(function(){return Mousetrap})}})(window,document)},{}],4:[function(require,module,exports){(function(process){function normalizeArray(parts,allowAboveRoot){var up=0;for(var i=parts.length-1;i>=0;i--){var last=parts[i];if(last==="."){parts.splice(i,1)}else if(last===".."){parts.splice(i,1);up++}else if(up){parts.splice(i,1);up--}}if(allowAboveRoot){for(;up--;up){parts.unshift("..")}}return parts}var splitPathRe=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;var splitPath=function(filename){return splitPathRe.exec(filename).slice(1)};exports.resolve=function(){var resolvedPath="",resolvedAbsolute=false;for(var i=arguments.length-1;i>=-1&&!resolvedAbsolute;i--){var path=i>=0?arguments[i]:process.cwd();if(typeof path!=="string"){throw new TypeError("Arguments to path.resolve must be strings")}else if(!path){continue}resolvedPath=path+"/"+resolvedPath;resolvedAbsolute=path.charAt(0)==="/"}resolvedPath=normalizeArray(filter(resolvedPath.split("/"),function(p){return!!p}),!resolvedAbsolute).join("/");return(resolvedAbsolute?"/":"")+resolvedPath||"."};exports.normalize=function(path){var isAbsolute=exports.isAbsolute(path),trailingSlash=substr(path,-1)==="/";path=normalizeArray(filter(path.split("/"),function(p){return!!p}),!isAbsolute).join("/");if(!path&&!isAbsolute){path="."}if(path&&trailingSlash){path+="/"}return(isAbsolute?"/":"")+path};exports.isAbsolute=function(path){return path.charAt(0)==="/"};exports.join=function(){var paths=Array.prototype.slice.call(arguments,0);return exports.normalize(filter(paths,function(p,index){if(typeof p!=="string"){throw new TypeError("Arguments to path.join must be strings")}return p}).join("/"))};exports.relative=function(from,to){from=exports.resolve(from).substr(1);to=exports.resolve(to).substr(1);function trim(arr){var start=0;for(;start=0;end--){if(arr[end]!=="")break}if(start>end)return[];return arr.slice(start,end-start+1)}var fromParts=trim(from.split("/"));var toParts=trim(to.split("/"));var length=Math.min(fromParts.length,toParts.length);var samePartsLength=length;for(var i=0;i1){for(var i=1;i= 0x80 (not a basic code point)","invalid-input":"Invalid input"},baseMinusTMin=base-tMin,floor=Math.floor,stringFromCharCode=String.fromCharCode,key;function error(type){throw RangeError(errors[type])}function map(array,fn){var length=array.length;var result=[];while(length--){result[length]=fn(array[length])}return result}function mapDomain(string,fn){var parts=string.split("@");var result="";if(parts.length>1){result=parts[0]+"@";string=parts[1]}string=string.replace(regexSeparators,".");var labels=string.split(".");var encoded=map(labels,fn).join(".");return result+encoded}function ucs2decode(string){var output=[],counter=0,length=string.length,value,extra;while(counter=55296&&value<=56319&&counter65535){value-=65536;output+=stringFromCharCode(value>>>10&1023|55296);value=56320|value&1023}output+=stringFromCharCode(value);return output}).join("")}function basicToDigit(codePoint){if(codePoint-48<10){return codePoint-22}if(codePoint-65<26){return codePoint-65}if(codePoint-97<26){return codePoint-97}return base}function digitToBasic(digit,flag){return digit+22+75*(digit<26)-((flag!=0)<<5)}function adapt(delta,numPoints,firstTime){var k=0;delta=firstTime?floor(delta/damp):delta>>1;delta+=floor(delta/numPoints);for(;delta>baseMinusTMin*tMax>>1;k+=base){delta=floor(delta/baseMinusTMin)}return floor(k+(baseMinusTMin+1)*delta/(delta+skew))}function decode(input){var output=[],inputLength=input.length,out,i=0,n=initialN,bias=initialBias,basic,j,index,oldi,w,k,digit,t,baseMinusT;basic=input.lastIndexOf(delimiter);if(basic<0){basic=0}for(j=0;j=128){error("not-basic")}output.push(input.charCodeAt(j))}for(index=basic>0?basic+1:0;index=inputLength){error("invalid-input")}digit=basicToDigit(input.charCodeAt(index++));if(digit>=base||digit>floor((maxInt-i)/w)){error("overflow")}i+=digit*w;t=k<=bias?tMin:k>=bias+tMax?tMax:k-bias;if(digitfloor(maxInt/baseMinusT)){error("overflow")}w*=baseMinusT}out=output.length+1;bias=adapt(i-oldi,out,oldi==0);if(floor(i/out)>maxInt-n){error("overflow")}n+=floor(i/out);i%=out;output.splice(i++,0,n)}return ucs2encode(output)}function encode(input){var n,delta,handledCPCount,basicLength,bias,j,m,q,k,t,currentValue,output=[],inputLength,handledCPCountPlusOne,baseMinusT,qMinusT;input=ucs2decode(input);inputLength=input.length;n=initialN;delta=0;bias=initialBias;for(j=0;j=n&¤tValuefloor((maxInt-delta)/handledCPCountPlusOne)){error("overflow")}delta+=(m-n)*handledCPCountPlusOne;n=m;for(j=0;jmaxInt){error("overflow")}if(currentValue==n){for(q=delta,k=base;;k+=base){t=k<=bias?tMin:k>=bias+tMax?tMax:k-bias;if(q0&&len>maxKeys){len=maxKeys}for(var i=0;i=0){kstr=x.substr(0,idx);vstr=x.substr(idx+1)}else{kstr=x;vstr=""}k=decodeURIComponent(kstr);v=decodeURIComponent(vstr);if(!hasOwnProperty(obj,k)){obj[k]=v}else if(isArray(obj[k])){obj[k].push(v)}else{obj[k]=[obj[k],v]}}return obj};var isArray=Array.isArray||function(xs){return Object.prototype.toString.call(xs)==="[object Array]"}},{}],8:[function(require,module,exports){"use strict";var stringifyPrimitive=function(v){switch(typeof v){case"string":return v;case"boolean":return v?"true":"false";case"number":return isFinite(v)?v:"";default:return""}};module.exports=function(obj,sep,eq,name){sep=sep||"&";eq=eq||"=";if(obj===null){obj=undefined}if(typeof obj==="object"){return map(objectKeys(obj),function(k){var ks=encodeURIComponent(stringifyPrimitive(k))+eq;if(isArray(obj[k])){return map(obj[k],function(v){return ks+encodeURIComponent(stringifyPrimitive(v))}).join(sep)}else{return ks+encodeURIComponent(stringifyPrimitive(obj[k]))}}).join(sep)}if(!name)return"";return encodeURIComponent(stringifyPrimitive(name))+eq+encodeURIComponent(stringifyPrimitive(obj))};var isArray=Array.isArray||function(xs){return Object.prototype.toString.call(xs)==="[object Array]"};function map(xs,f){if(xs.map)return xs.map(f);var res=[];for(var i=0;i",'"',"`"," ","\r","\n","\t"],unwise=["{","}","|","\\","^","`"].concat(delims),autoEscape=["'"].concat(unwise),nonHostChars=["%","/","?",";","#"].concat(autoEscape),hostEndingChars=["/","?","#"],hostnameMaxLen=255,hostnamePartPattern=/^[a-z0-9A-Z_-]{0,63}$/,hostnamePartStart=/^([a-z0-9A-Z_-]{0,63})(.*)$/,unsafeProtocol={javascript:true,"javascript:":true},hostlessProtocol={javascript:true,"javascript:":true},slashedProtocol={http:true,https:true,ftp:true,gopher:true,file:true,"http:":true,"https:":true,"ftp:":true,"gopher:":true,"file:":true},querystring=require("querystring");function urlParse(url,parseQueryString,slashesDenoteHost){if(url&&isObject(url)&&url instanceof Url)return url;var u=new Url;u.parse(url,parseQueryString,slashesDenoteHost);return u}Url.prototype.parse=function(url,parseQueryString,slashesDenoteHost){if(!isString(url)){throw new TypeError("Parameter 'url' must be a string, not "+typeof url)}var rest=url;rest=rest.trim();var proto=protocolPattern.exec(rest);if(proto){proto=proto[0];var lowerProto=proto.toLowerCase();this.protocol=lowerProto;rest=rest.substr(proto.length)}if(slashesDenoteHost||proto||rest.match(/^\/\/[^@\/]+@[^@\/]+/)){var slashes=rest.substr(0,2)==="//";if(slashes&&!(proto&&hostlessProtocol[proto])){rest=rest.substr(2);this.slashes=true}}if(!hostlessProtocol[proto]&&(slashes||proto&&!slashedProtocol[proto])){var hostEnd=-1;for(var i=0;i127){newpart+="x"}else{newpart+=part[j]}}if(!newpart.match(hostnamePartPattern)){var validParts=hostparts.slice(0,i);var notHost=hostparts.slice(i+1);var bit=part.match(hostnamePartStart);if(bit){validParts.push(bit[1]);notHost.unshift(bit[2])}if(notHost.length){rest="/"+notHost.join(".")+rest}this.hostname=validParts.join(".");break}}}}if(this.hostname.length>hostnameMaxLen){this.hostname=""}else{this.hostname=this.hostname.toLowerCase()}if(!ipv6Hostname){var domainArray=this.hostname.split(".");var newOut=[];for(var i=0;i0?result.host.split("@"):false;if(authInHost){result.auth=authInHost.shift();result.host=result.hostname=authInHost.shift()}}result.search=relative.search;result.query=relative.query;if(!isNull(result.pathname)||!isNull(result.search)){result.path=(result.pathname?result.pathname:"")+(result.search?result.search:"")}result.href=result.format();return result}if(!srcPath.length){result.pathname=null;if(result.search){result.path="/"+result.search}else{result.path=null}result.href=result.format();return result}var last=srcPath.slice(-1)[0];var hasTrailingSlash=(result.host||relative.host)&&(last==="."||last==="..")||last==="";var up=0;for(var i=srcPath.length;i>=0;i--){last=srcPath[i];if(last=="."){srcPath.splice(i,1)}else if(last===".."){srcPath.splice(i,1);up++}else if(up){srcPath.splice(i,1);up--}}if(!mustEndAbs&&!removeAllDots){for(;up--;up){srcPath.unshift("..")}}if(mustEndAbs&&srcPath[0]!==""&&(!srcPath[0]||srcPath[0].charAt(0)!=="/")){srcPath.unshift("")}if(hasTrailingSlash&&srcPath.join("/").substr(-1)!=="/"){srcPath.push("")}var isAbsolute=srcPath[0]===""||srcPath[0]&&srcPath[0].charAt(0)==="/";if(psychotic){result.hostname=result.host=isAbsolute?"":srcPath.length?srcPath.shift():"";var authInHost=result.host&&result.host.indexOf("@")>0?result.host.split("@"):false;if(authInHost){result.auth=authInHost.shift();result.host=result.hostname=authInHost.shift()}}mustEndAbs=mustEndAbs||result.host&&srcPath.length;if(mustEndAbs&&!isAbsolute){srcPath.unshift("")}if(!srcPath.length){result.pathname=null;result.path=null}else{result.pathname=srcPath.join("/")}if(!isNull(result.pathname)||!isNull(result.search)){result.path=(result.pathname?result.pathname:"")+(result.search?result.search:"")}result.auth=relative.auth||result.auth;result.slashes=result.slashes||relative.slashes;result.href=result.format();return result};Url.prototype.parseHost=function(){var host=this.host;var port=portPattern.exec(host);if(port){port=port[0];if(port!==":"){this.port=port.substr(1)}host=host.substr(0,host.length-port.length)}if(host)this.hostname=host};function isString(arg){return typeof arg==="string"}function isObject(arg){return typeof arg==="object"&&arg!==null}function isNull(arg){return arg===null}function isNullOrUndefined(arg){return arg==null}},{punycode:6,querystring:9}],11:[function(require,module,exports){var $=require("jquery");function toggleDropdown(e){var $dropdown=$(e.currentTarget).parent().find(".dropdown-menu");$dropdown.toggleClass("open");e.stopPropagation();e.preventDefault()}function closeDropdown(e){$(".dropdown-menu").removeClass("open")}function init(){$(document).on("click",".toggle-dropdown",toggleDropdown);$(document).on("click",".dropdown-menu",function(e){e.stopPropagation()});$(document).on("click",closeDropdown)}module.exports={init:init}},{jquery:1}],12:[function(require,module,exports){var $=require("jquery");module.exports=$({})},{jquery:1}],13:[function(require,module,exports){var $=require("jquery");var _=require("lodash");var storage=require("./storage");var dropdown=require("./dropdown");var events=require("./events");var state=require("./state");var keyboard=require("./keyboard");var navigation=require("./navigation");var sidebar=require("./sidebar");var toolbar=require("./toolbar");function start(config){sidebar.init();keyboard.init();dropdown.init();navigation.init();toolbar.createButton({index:0,icon:"fa fa-align-justify",label:"Toggle Sidebar",onClick:function(e){e.preventDefault();sidebar.toggle()}});events.trigger("start",config);navigation.notify()}var gitbook={start:start,events:events,state:state,toolbar:toolbar,sidebar:sidebar,storage:storage,keyboard:keyboard};var MODULES={gitbook:gitbook,jquery:$,lodash:_};window.gitbook=gitbook;window.$=$;window.jQuery=$;gitbook.require=function(mods,fn){mods=_.map(mods,function(mod){mod=mod.toLowerCase();if(!MODULES[mod]){throw new Error("GitBook module "+mod+" doesn't exist")}return MODULES[mod]});fn.apply(null,mods)};module.exports={}},{"./dropdown":11,"./events":12,"./keyboard":14,"./navigation":16,"./sidebar":18,"./state":19,"./storage":20,"./toolbar":21,jquery:1,lodash:2}],14:[function(require,module,exports){var Mousetrap=require("mousetrap");var navigation=require("./navigation");var sidebar=require("./sidebar");function bindShortcut(keys,fn){Mousetrap.bind(keys,function(e){fn();return false})}function init(){bindShortcut(["right"],function(e){navigation.goNext()});bindShortcut(["left"],function(e){navigation.goPrev()});bindShortcut(["s"],function(e){sidebar.toggle()})}module.exports={init:init,bind:bindShortcut}},{"./navigation":16,"./sidebar":18,mousetrap:3}],15:[function(require,module,exports){var state=require("./state");function showLoading(p){state.$book.addClass("is-loading");p.always(function(){state.$book.removeClass("is-loading")});return p}module.exports={show:showLoading}},{"./state":19}],16:[function(require,module,exports){var $=require("jquery");var url=require("url");var events=require("./events");var state=require("./state");var loading=require("./loading");var usePushState=typeof history.pushState!=="undefined";function handleNavigation(relativeUrl,push){var uri=url.resolve(window.location.pathname,relativeUrl);notifyPageChange();location.href=relativeUrl;return}function updateNavigationPosition(){var bodyInnerWidth,pageWrapperWidth;bodyInnerWidth=parseInt($(".body-inner").css("width"),10);pageWrapperWidth=parseInt($(".page-wrapper").css("width"),10);$(".navigation-next").css("margin-right",bodyInnerWidth-pageWrapperWidth+"px")}function notifyPageChange(){events.trigger("page.change")}function preparePage(notify){var $bookBody=$(".book-body");var $bookInner=$bookBody.find(".body-inner");var $pageWrapper=$bookInner.find(".page-wrapper");updateNavigationPosition();$bookInner.scrollTop(0);$bookBody.scrollTop(0);if(notify!==false)notifyPageChange()}function isLeftClickEvent(e){return e.button===0}function isModifiedEvent(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}function handlePagination(e){if(isModifiedEvent(e)||!isLeftClickEvent(e)){return}e.stopPropagation();e.preventDefault();var url=$(this).attr("href");if(url)handleNavigation(url,true)}function goNext(){var url=$(".navigation-next").attr("href");if(url)handleNavigation(url,true)}function goPrev(){var url=$(".navigation-prev").attr("href");if(url)handleNavigation(url,true)}function init(){$.ajaxSetup({});if(location.protocol!=="file:"){history.replaceState({path:window.location.href},"")}window.onpopstate=function(event){if(event.state===null){return}return handleNavigation(event.state.path,false)};$(document).on("click",".navigation-prev",handlePagination);$(document).on("click",".navigation-next",handlePagination);$(document).on("click",".summary [data-path] a",handlePagination);$(window).resize(updateNavigationPosition);preparePage(false)}module.exports={init:init,goNext:goNext,goPrev:goPrev,notify:notifyPageChange}},{"./events":12,"./loading":15,"./state":19,jquery:1,url:10}],17:[function(require,module,exports){module.exports={isMobile:function(){return document.body.clientWidth<=600}}},{}],18:[function(require,module,exports){var $=require("jquery");var _=require("lodash");var storage=require("./storage");var platform=require("./platform");var state=require("./state");function toggleSidebar(_state,animation){if(state!=null&&isOpen()==_state)return;if(animation==null)animation=true;state.$book.toggleClass("without-animation",!animation);state.$book.toggleClass("with-summary",_state);storage.set("sidebar",isOpen())}function isOpen(){return state.$book.hasClass("with-summary")}function init(){if(platform.isMobile()){toggleSidebar(false,false)}else{toggleSidebar(storage.get("sidebar",true),false)}$(document).on("click",".book-summary li.chapter a",function(e){if(platform.isMobile())toggleSidebar(false,false)})}function filterSummary(paths){var $summary=$(".book-summary");$summary.find("li").each(function(){var path=$(this).data("path");var st=paths==null||_.contains(paths,path);$(this).toggle(st);if(st)$(this).parents("li").show()})}module.exports={init:init,isOpen:isOpen,toggle:toggleSidebar,filter:filterSummary}},{"./platform":17,"./state":19,"./storage":20,jquery:1,lodash:2}],19:[function(require,module,exports){var $=require("jquery");var url=require("url");var path=require("path");var state={};state.update=function(dom){var $book=$(dom.find(".book"));state.$book=$book;state.level=$book.data("level");state.basePath=$book.data("basepath");state.innerLanguage=$book.data("innerlanguage");state.revision=$book.data("revision");state.filepath=$book.data("filepath");state.chapterTitle=$book.data("chapter-title");state.root=url.resolve(location.protocol+"//"+location.host,path.dirname(path.resolve(location.pathname.replace(/\/$/,"/index.html"),state.basePath))).replace(/\/?$/,"/");state.bookRoot=state.innerLanguage?url.resolve(state.root,".."):state.root};state.update($);module.exports=state},{jquery:1,path:4,url:10}],20:[function(require,module,exports){var baseKey="";module.exports={setBaseKey:function(key){baseKey=key},set:function(key,value){key=baseKey+":"+key;try{sessionStorage[key]=JSON.stringify(value)}catch(e){}},get:function(key,def){key=baseKey+":"+key;if(sessionStorage[key]===undefined)return def;try{var v=JSON.parse(sessionStorage[key]);return v==null?def:v}catch(err){return sessionStorage[key]||def}},remove:function(key){key=baseKey+":"+key;sessionStorage.removeItem(key)}}},{}],21:[function(require,module,exports){var $=require("jquery");var _=require("lodash");var events=require("./events");var buttons=[];function insertAt(parent,selector,index,element){var lastIndex=parent.children(selector).length;if(index<0){index=Math.max(0,lastIndex+1+index)}parent.append(element);if(index",{class:"dropdown-menu",html:'
'});if(_.isString(dropdown)){$menu.append(dropdown)}else{var groups=_.map(dropdown,function(group){if(_.isArray(group))return group;else return[group]});_.each(groups,function(group){var $group=$("
",{class:"buttons"});var sizeClass="size-"+group.length;_.each(group,function(btn){btn=_.defaults(btn||{},{text:"",className:"",onClick:defaultOnClick});var $btn=$("'; + var clipboard; + + gitbook.events.bind("page.change", function() { + + if (!ClipboardJS.isSupported()) return; + + // the page.change event is thrown twice: before and after the page changes + if (clipboard) { + // clipboard is already defined but we are on the same page + if (clipboard._prevPage === window.location.pathname) return; + // clipboard is already defined and url path change + // we can deduct that we are before page changes + clipboard.destroy(); // destroy the previous events listeners + clipboard = undefined; // reset the clipboard object + return; + } + + $(copyButton).prependTo("div.sourceCode"); + + clipboard = new ClipboardJS(".copy-to-clipboard-button", { + text: function(trigger) { + return trigger.parentNode.textContent; + } + }); + + clipboard._prevPage = window.location.pathname + + }); + +}); diff --git a/docs/libs/gitbook-2.6.7/js/plugin-fontsettings.js b/docs/libs/gitbook-2.6.7/js/plugin-fontsettings.js new file mode 100644 index 0000000000..a70f0fb374 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/js/plugin-fontsettings.js @@ -0,0 +1,152 @@ +gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { + var fontState; + + var THEMES = { + "white": 0, + "sepia": 1, + "night": 2 + }; + + var FAMILY = { + "serif": 0, + "sans": 1 + }; + + // Save current font settings + function saveFontSettings() { + gitbook.storage.set("fontState", fontState); + update(); + } + + // Increase font size + function enlargeFontSize(e) { + e.preventDefault(); + if (fontState.size >= 4) return; + + fontState.size++; + saveFontSettings(); + }; + + // Decrease font size + function reduceFontSize(e) { + e.preventDefault(); + if (fontState.size <= 0) return; + + fontState.size--; + saveFontSettings(); + }; + + // Change font family + function changeFontFamily(index, e) { + e.preventDefault(); + + fontState.family = index; + saveFontSettings(); + }; + + // Change type of color + function changeColorTheme(index, e) { + e.preventDefault(); + + var $book = $(".book"); + + if (fontState.theme !== 0) + $book.removeClass("color-theme-"+fontState.theme); + + fontState.theme = index; + if (fontState.theme !== 0) + $book.addClass("color-theme-"+fontState.theme); + + saveFontSettings(); + }; + + function update() { + var $book = gitbook.state.$book; + + $(".font-settings .font-family-list li").removeClass("active"); + $(".font-settings .font-family-list li:nth-child("+(fontState.family+1)+")").addClass("active"); + + $book[0].className = $book[0].className.replace(/\bfont-\S+/g, ''); + $book.addClass("font-size-"+fontState.size); + $book.addClass("font-family-"+fontState.family); + + if(fontState.theme !== 0) { + $book[0].className = $book[0].className.replace(/\bcolor-theme-\S+/g, ''); + $book.addClass("color-theme-"+fontState.theme); + } + }; + + function init(config) { + var $bookBody, $book; + + //Find DOM elements. + $book = gitbook.state.$book; + $bookBody = $book.find(".book-body"); + + // Instantiate font state object + fontState = gitbook.storage.get("fontState", { + size: config.size || 2, + family: FAMILY[config.family || "sans"], + theme: THEMES[config.theme || "white"] + }); + + update(); + }; + + + gitbook.events.bind("start", function(e, config) { + var opts = config.fontsettings; + if (!opts) return; + + // Create buttons in toolbar + gitbook.toolbar.createButton({ + icon: 'fa fa-font', + label: 'Font Settings', + className: 'font-settings', + dropdown: [ + [ + { + text: 'A', + className: 'font-reduce', + onClick: reduceFontSize + }, + { + text: 'A', + className: 'font-enlarge', + onClick: enlargeFontSize + } + ], + [ + { + text: 'Serif', + onClick: _.partial(changeFontFamily, 0) + }, + { + text: 'Sans', + onClick: _.partial(changeFontFamily, 1) + } + ], + [ + { + text: 'White', + onClick: _.partial(changeColorTheme, 0) + }, + { + text: 'Sepia', + onClick: _.partial(changeColorTheme, 1) + }, + { + text: 'Night', + onClick: _.partial(changeColorTheme, 2) + } + ] + ] + }); + + + // Init current settings + init(opts); + }); +}); + + diff --git a/docs/libs/gitbook-2.6.7/js/plugin-search.js b/docs/libs/gitbook-2.6.7/js/plugin-search.js new file mode 100644 index 0000000000..747fccebd0 --- /dev/null +++ b/docs/libs/gitbook-2.6.7/js/plugin-search.js @@ -0,0 +1,270 @@ +gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { + var index = null; + var fuse = null; + var _search = {engine: 'lunr', opts: {}}; + var $searchInput, $searchLabel, $searchForm; + var $highlighted = [], hi, hiOpts = { className: 'search-highlight' }; + var collapse = false, toc_visible = []; + + function init(config) { + // Instantiate search settings + _search = gitbook.storage.get("search", { + engine: config.search.engine || 'lunr', + opts: config.search.options || {}, + }); + }; + + // Save current search settings + function saveSearchSettings() { + gitbook.storage.set("search", _search); + } + + // Use a specific index + function loadIndex(data) { + // [Yihui] In bookdown, I use a character matrix to store the chapter + // content, and the index is dynamically built on the client side. + // Gitbook prebuilds the index data instead: https://github.com/GitbookIO/plugin-search + // We can certainly do that via R packages V8 and jsonlite, but let's + // see how slow it really is before improving it. On the other hand, + // lunr cannot handle non-English text very well, e.g. the default + // tokenizer cannot deal with Chinese text, so we may want to replace + // lunr with a dumb simple text matching approach. + if (_search.engine === 'lunr') { + index = lunr(function () { + this.ref('url'); + this.field('title', { boost: 10 }); + this.field('body'); + }); + data.map(function(item) { + index.add({ + url: item[0], + title: item[1], + body: item[2] + }); + }); + return; + } + fuse = new Fuse(data.map((_data => { + return { + url: _data[0], + title: _data[1], + body: _data[2] + }; + })), Object.assign( + { + includeScore: true, + threshold: 0.1, + ignoreLocation: true, + keys: ["title", "body"] + }, + _search.opts + )); + } + + // Fetch the search index + function fetchIndex() { + return $.getJSON(gitbook.state.basePath+"/search_index.json") + .then(loadIndex); // [Yihui] we need to use this object later + } + + // Search for a term and return results + function search(q) { + let results = []; + switch (_search.engine) { + case 'fuse': + if (!fuse) return; + results = fuse.search(q).map(function(result) { + var parts = result.item.url.split('#'); + return { + path: parts[0], + hash: parts[1] + }; + }); + break; + case 'lunr': + default: + if (!index) return; + results = _.chain(index.search(q)).map(function(result) { + var parts = result.ref.split("#"); + return { + path: parts[0], + hash: parts[1] + }; + }) + .value(); + } + + // [Yihui] Highlight the search keyword on current page + $highlighted = $('.page-inner') + .unhighlight(hiOpts).highlight(q, hiOpts).find('span.search-highlight'); + scrollToHighlighted(0); + + return results; + } + + // [Yihui] Scroll the chapter body to the i-th highlighted string + function scrollToHighlighted(d) { + var n = $highlighted.length; + hi = hi === undefined ? 0 : hi + d; + // navignate to the previous/next page in the search results if reached the top/bottom + var b = hi < 0; + if (d !== 0 && (b || hi >= n)) { + var path = currentPath(), n2 = toc_visible.length; + if (n2 === 0) return; + for (var i = b ? 0 : n2; (b && i < n2) || (!b && i >= 0); i += b ? 1 : -1) { + if (toc_visible.eq(i).data('path') === path) break; + } + i += b ? -1 : 1; + if (i < 0) i = n2 - 1; + if (i >= n2) i = 0; + var lnk = toc_visible.eq(i).find('a[href$=".html"]'); + if (lnk.length) lnk[0].click(); + return; + } + if (n === 0) return; + var $p = $highlighted.eq(hi); + $p[0].scrollIntoView(); + $highlighted.css('background-color', ''); + // an orange background color on the current item and removed later + $p.css('background-color', 'orange'); + setTimeout(function() { + $p.css('background-color', ''); + }, 2000); + } + + function currentPath() { + var href = window.location.pathname; + href = href.substr(href.lastIndexOf('/') + 1); + return href === '' ? 'index.html' : href; + } + + // Create search form + function createForm(value) { + if ($searchForm) $searchForm.remove(); + if ($searchLabel) $searchLabel.remove(); + if ($searchInput) $searchInput.remove(); + + $searchForm = $('
', { + 'class': 'book-search', + 'role': 'search' + }); + + $searchLabel = $('",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0 + + + + + + Chapter 6 Managing Accessions | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 6 Managing Accessions

+

The “Manage Accession” page provides links for adding new accessions. +New accessions can be added to the database by either using a List or by uploading an Excel file (either XLS or XLSX format). +Both options are explained in more detail below. +To begin, click on the “Add Accessions or Upload Accession Info” link.

+

+

This will open a dialog allowing you to select either “Using Lists” or “Uploading a File”.

+
+

6.1 Add Accessions Using A List

+

First we will show how to add accessions “Using Lists”.

+

+

Here you select an accession list which you have previously made (see List Manager chapter). +If you need to create or edit your list you can do so now by clicking “Manage Lists”. +After selecting your list, click “Continue”.

+

The contents of the list will be checked against the database, and elements that are already present will be flagged. +A dialog will appear that will show the accessions which already exist in the database.

+

+

After clicking on the “Continue” button, the next dialog will appear with accessions that have very similar names as the accession that you are adding. +In the example below, there are two accession with very similar names to accessions already in the database. +‘TME0419’ is very similar to ‘TME419’, and probably represent the same line, so it would be a mistake to add this the database again. +Duplicate lines in the database should be avoided, as they cause problems when evaluating lines; data is divided up among several duplicates, making it harder to get the full picture about an accession.

+

+

To avoid situations in adding a mistaken duplicate accession, the database gives you options for moving forward with these very similar looking accession names. +You can either “continue saving the name in your list”, “replace name in your list with selected existing name”, “remove name in your list and ignore”, or “add name in your list as a synonym to selected existing name”.

+

+

Clicking “Download Fuzzy Matches” will return a tabular result of the “fuzzy” accession name results shown. +Click “Make changes and continue” to move on.

+

The final dialog shows the accessions that will be added. +Here you need to assign the species of these accessions. +You can optionally group the accessions into a population and/or add an organization for the accessions.

+

+

Once you click “Add Accessions”, the new accessions will be created in the database and you will see the following confirmation dialog, which includes links to the newly created accessions.

+

+
+
+

6.2 Uploading Accessions and Accession’s Info From A File

+

Uploading accessions using a file is very similar to using a list, but enables you to add a variety of attributes, such as synonyms or ploidy levels, to the accessions in bulk.

+

+

Clicking on “Spreadsheet format” will show the required structure of the spreadsheet. +The file must be XLS or XLSX format and can contain a number of header columns as attributes. +It is important that you use exactly the same header column names as listed here. +In columns that indicate that many attribute values can be passed at once using (s), such as synonym(s), you can pass a comma separated list of values, such as ‘synonym1,synonym2’.

+

+

Once you have selected your XLS or XLSX file for upload, click “Continue”.

+

The following process is the same way as with lists:

+

The first dialog which can appear will show accession names which are already in the database.

+

Click “Continue” and the next dialog that can appear will show “fuzzy” matches for the accession names you are trying to upload. +Here you can choose to prevent adding accession names which look very similar to each other as wrongly duplicated accessions.

+

Click “Continue” and the final dialog that will appear will show the information to be added into the database. +Here it is divided into accession names that are new and accession names that already exist in the database; however, for the accession names that already exist it will show additional attributes that originated from your file that will be added to these accessions.

+

+

Once you click “Add Accessions”, the new accessions and information will be created in the database and you will see the following confirmation dialog, which includes links to the created and updated accessions.

+

+
+
+

6.3 Email alert for accession upload

+

When uploading a large number of accessions from a file, uploads can take a while, as the system needs to perform a series of checks on each entry. +You have the option to receive an email notification about the status and results of your upload by clicking the “Email Alert” checkbox. +By default, the system will use the email address associated with your account, but you have the option of entering a different email address if you prefer. +After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. +Once the process completes, you will receive an email with the upload results, including any warnings or errors that may have occurred during the upload.

+

+
+
+

6.4 Add Parentage (Pedigree) Information to Accessions

+

Pedigree data can be uploaded from your computer by clicking on “Upload Pedigree File”

+

+

You can find detailed information on how to prepare pedigree file by clicking on “File format information”. +The file format can be tab or comma delimited text file, or Excel files (.xls or .xlsx).

+

The currently supported format has four columns:

+

progeny name female parent accession male parent accession type

+

Type can be biparental, self, backcross, sib, polycross, reselected, or open. +In the case of the open type, the male parent accession field can remain blank. +For all other types, both columns should be filled, even if they contain the same information as another column (such as self).

+

+

+
+
+

6.5 Working with grafts

+

Grafts are plants that are composed of a rootstock and a scion, which are genetically different and fused together, usually at the stem level.

+

To work with grafts, the grafts interface needs to be activated by a system administrator. +Please contact your Breedbase provider. +Briefly, a configuration parameter needs to be added to the sgn_local.conf file, show_grafting_interface. +It should be set to 1 in sgn_local.conf, the default is 0 in sgn.conf.

+

Grafts to be created need to be specified using an Excel file (xlsx format) with two columns. +The first column should have the header “scion accession” and should list accession names that will be scions. +The second column should have the header “rootstock accession” and should list accession names that will be rootstocks.

+

In the database, the graft accessions will be created as single accessions. +The graft accession will have two relationships, one to the scion accession (scion_of relationship) and one to the rootstock (rootstock_of relationship). +These relationships are displayed on the pedigree viewer. +The graft accession name is created from the scion accession name and the rootstock accession name, separated by the graft separator character. +By default, the graft separator character is the plus sign ‘+’. +The graft separator character can be changed in the sgn_local.conf file, using the parameter graft_separator_string. +The graft separator string should not occur in any other accession names that are not grafts.

+

When the grafting interface is activated, a new button will be shown on the manage accessions page, called “Upload Grafts”.

+

Clicking the button brings up the upload grafts dialog.

+

Select the Excel file containing the grafting information. +The system will validate the file, for example, check whether the accessions are in the database, and if the headers are correct.

+

The validation result will be presented, and if problems are found, they will be listed. +In addition, if there are problems, the Upload button will be grayed out and upload will not be possible. +Conversely, if there are no problems, the Upload button will be activated and can be clicked to store the data.

+

If the upload completes, a completion message is displayed with a summary what was uploaded.

+

Grafted accessions can be used like any other accession, for example, they can be used on field layouts. +If you create a list of graft accessions, use the list type ‘accessions’.

+

Note that you shouldn’t create new grafts based on other grafts. +The scion accession and the rootstock accession have to be different, otherwise they will not be created.

+
+
+

6.6 Bulk renaming of accessions

+

Accessions can be renamed in bulk using the rename accessions feature. +To rename accessions, prepare a tab delimited file with two columns: the first column should have the header “old name” and contain the accession names that need to be changed. +The second column should have the header “new name” and contain the names that the accessions in column 1 should be renamed to.

+

The accession renaming feature is available from the Manage->Accessions page. +Click on the “Rename Accessions” button. +The first step is the upload of the file with a verification step. +The verification step checks whether all the accession names in column 1 exist in the database, and whether all the accession names given in column 2 do NOT exist in the database. +Only if both conditions are met, will the “rename” button become active, otherwise an error message is displayed listing the offending accession names.

+

Optionally, the old name can be automatically added as a synonym to the renamed accession, using the checkbox on the submit form. +This option is clicked by default. +Unclick the checkbox to NOT save any old names as synonyms.

+

Note that accession renaming should not be undertaken lightly. +This feature is intended for special use cases, such as where accessions are created in a nursery with a name that is different from the accession name in the downstream breeding program.

+

It can also be used to rename accessions in bulk that have spelling mistakes and other issues. +Please note however, that the tool does not make any attempt to change the names of associated elements, such a plots, that may have been constructed using accession names.

+

Because of the many implications of accession renaming, the feature is limited to accounts with the curator role.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-barcodes.html b/docs/managing-barcodes.html new file mode 100644 index 0000000000..fe026cfe06 --- /dev/null +++ b/docs/managing-barcodes.html @@ -0,0 +1,495 @@ + + + + + + + Chapter 15 Managing Barcodes | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 15 Managing Barcodes

+

SGN databases provide tools for generating barcodes for stock identification. To access “Barcode Tools” page, clicking on “Barcodes” in the “Manage” menu.

+

+

“Barcode Tools” page provides four options for generating barcodes:

+
    +

  • Single barcode

    • +

  • Multiple barcodes

    • +

  • Plot phenotyping barcodes

    • +

  • Trial barcodes

    • +
+

To generate single barcode, clicking on “Generate Barcode” link on the “Barcode Tools” page.

+

+

In the “Generate Barcode” section, specify the name of the barcode, size of the barcode, then clicking on “Generate Barcode”

+

+

The database will generate a barcode for your stock. The barcode can be printed for your stock identification. It also appears on its corresponding stock page.

+

+

If you have a list of stocks that you want to generate barcodes, you can use “Download Stock Barcodes” section. You have three options for entering stock names:

+
    +

  1. Typing in stock names, or copy and paste from other file into the box (1)

    2. +

  2. Choosing a list of stocks from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button.

    3. +

  3. Uploading a “Tab-delimited Text File” with stock names.

    4. +

  4. Select an optional printing format from the available formats.

    5. +
+

You can select printer settings that you prefer in the “Printer Settings” section. After you enter stock names and specify printer settings, clicking on “Download Barcodes” button at the bottom of the page.

+

+

If you have a list of plots that you want to generate phenotyping barcodes, you can use “Download Plot Phenotyping Barcodes” section. You have three options for entering plot names:

+
    +

  1. Typing in plot names, or copy and paste from other file into the box (1)

    2. +

  2. Choosing a list of plots from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button.

    3. +

  3. Uploading a “Tab-delimited Text File” with plot names.

    4. +
+

+

If you have a list of trials that you want to generate barcodes, you can use “Download Trial Barcodes” section. You have three options for entering trial names:

+
    +

  1. Typing in trial names, or copy and paste from other file into the box (1)

    2. +

  2. Choosing a list of trial from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button.

    3. +

  3. Uploading a “Tab-delimited Text File” with trial names.

    4. +
+

+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-breeding-programs.html b/docs/managing-breeding-programs.html new file mode 100644 index 0000000000..0d969d2de3 --- /dev/null +++ b/docs/managing-breeding-programs.html @@ -0,0 +1,461 @@ + + + + + + + Chapter 4 Managing Breeding Programs | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 4 Managing Breeding Programs

+

New breeding programs can be added by using “Add New Program” button on the “Manage Breeding Programs” page.

+

+

Clicking on the “Add New Program” button will generate a blank form for you to fill out the name and description of the breeding program that you want to add. After completing the form, click on “Add Breeding Program” button to finish the process.

+

+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-crosses.html b/docs/managing-crosses.html new file mode 100644 index 0000000000..bc3065a19e --- /dev/null +++ b/docs/managing-crosses.html @@ -0,0 +1,575 @@ + + + + + + + Chapter 9 Managing Crosses | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 9 Managing Crosses

+

Information for crosses can be managed using the “Crosses” option in the Manage menu.

+

+
+

9.1 Crossing Experiment

+

Different crosses in the same trial/nursery/project are grouped via “crossing experiment”. Crossing experiments are organized based on their breeding programs. To find a crossing experiment, you can either type the crossing experiment name in the “Search” box, or look for the crossing experiment directly in its breeding program by clicking on the “+” icon. In each breeding program, crossing experiments can be placed directly in the breeding program, or organized in folders. The “Folders” section allows you to place crossing experiments in folders, move a crossing experiment in a folder to another folder, or rearrange your folders within a breeding program.

+

+

+
+

9.1.1 Add New Crossing Experiment

+

To add a new crossing experiment, click on “Add Crossing Experiment” link. +

+

Required Information:

+

• “Crossing Experiment Name”: enter a name for the crossing experiment. The crossing experiment name must not already exist in the database.

+

• “Breeding program”: select a breeding program that is available in the database. New breeding programs can be added on the “Breeding program” page, accessible from the “Manage” menu. Breeding Program Page

+

• “Location”: select a location for the crossing experiment. New locations can be entered on the “Locations” page, accessible from the “Manage” menu. Location Page

+

• “Year”: select a year.

+

• “Description”: enter a description for the crossing experiment.

+

After filling in the information, click “Submit” to generate the crossing experiment.

+

+
+
+
+

9.2 Cross

+
+

9.2.1 Add New Crosses

+
+

Add a cross by using the “Add New Cross” dialog

+

To add a single new cross, click on “Add Cross” link.

+

+

Enter cross information in the popup dialog.

+

+

Required Information:

+

• “Crossing experiment”: select a crossing experiment available in the database.

+

• “Location”: select a location available in the database.

+

• “Cross name”: enter a name for the cross. The cross name must not already exist in the database.

+

• “Cross type”: the options for cross types are: biparental, self, open pollinated, bulk, bulk selfed, bulk and open pollinated, double haploid, polycross, reciprocal and multicross.

+

+

• The “Female Parent” and “Male Parent” field are auto-complete fields for accessions that are already in the database. The parents specified will be entered in the pedigree of the new accessions generated by this cross.

+

Optional Information:

+

• “Female Plot and/or Male Plot”: In addition to the accession names, specific plots used in the cross can also be added to the database. To retrieve plot names associated with each female/male accession, enter your trial name, then click “Search Plots”. Plot names of each parental accession in that field trial will be shown in the drop-down list, you can then select the plot used in the cross.

+

+

Additional crossing experimental information such as pollination date, number of flowers, number of fruits, number of seeds can be specified during adding new cross. Alternatively, this information can be updated or edited directly on the “Cross Details” page.

+

If you know the number of accessions that are generated from the cross, they can be instantiated immediately in the database by clicking the “Add accessions for progeny” checkbox and specifying the number.

+

+

Click “Submit” to generate the cross.

+
+
+

Upload New Crosses

+

To upload new crosses from an Excel file (.xls or .xlsx), click on “Upload Crosses” link.

+

+

Select a crossing experiment and a location available in the database from drop-down lists and choose a file that you want to upload, then click “Upload File”.

+

+

Please check spreadsheet format carefully. The file must be an Excel file (.xls or .xlsx).

+

+
+
+
+

9.2.2 Update Crosses by Uploading

+

To upload progenies and/or experimental info of crosses already in the database, go to “Manage-Upload” page.

+

In the “Crosses” section, there are links for uploading progenies and experimental info.

+

+

Please check spreadsheet format in each link carefully. The file must be an Excel file (.xls or .xlsx).

+

+

+

Note: crossing experimental information is customized based on the need for each crop. As a result, column headers for experimental info in your database may be different from the information shown in this manual.

+
+
+
+

9.3 Cross Wishlist

+

An Android ODK application is being developed to record cross information on a mobile device in the field. To link this mobile application with the database, the Cross Wishlist can be used to create a plan for which crosses to perform.

+

This tool is available on the Manage Cross page. It is currently only available on certain databases, so when you click this link you may see an alert mentioning that the cross wishlist is not available on your database.

+

+
+

9.3.1 Create a Cross Wishlist

+
+

Step 1. Select the accessions to be crossed in your trial

+

There are two interfaces for this step, either “Not Using Lists” or “Using Lists”. Depending on if you already have a list of female and male accessions to use, you can decide on which interface to use. The end result of using either interface is the same.

+

+

We will start by showing “Not Using Lists”. First select the trial in which the crosses are to be performed. This will populate a select box with all the accessions used in that trial. From here, one or many accessions can be selected as the female accession.

+

+

Once the female accessions are selected, a table is populated. Each row in this table begins with the female accession that was selected, followed by a select box with all the accessions used in the trial. From here, one or many accessions can be selected as the male to use in the cross.

+

+

Once the male accessions are selected to cross with each female accession, a table indicating priorities appears. Priority is meant to indicate an order in which to attempt the cross; first the highest priority male will be considered, but if this cross is not possible then subsequent males will be considered. An equal priority can be given and this will not indicate a specific order to follow.

+

+

Alternatively, we could have used the “Using List” interface instead. Here we select the trial in which the crosses will be performed and we provide a list of accessions to consider for the females and the males to be crossed.

+

+
+
+

Step 2. Select the female plots to be considered in the crosses

+

After selecting your lists, the table below is populated. The first column has all the female accessions specified and the header row has all the male accessions specified. The males to consider crossing with each female are indicated with priority.

+

+

After female and male accessions are selected to cross, either by the “Nor Using List” or “Using List” interface, click Next. The next dialog will allow selection of specific female plots to use for the cross. Sections for each female accession selected will appear with the field layout displayed. Selecting all plots in which the female is present indicates that the cross should be performed on all plots where that female accession is present.

+

+
+
+

Step 3. Transfer the cross wishlist to your mobile crossing application

+

Clicking “Push Cross Wishlst for ODK Use” will send the cross wishlist plan to the ONA server for use by the mobile ODK application. Crosses can then be performed and recorded in the field using the mobile application. Afterwards, the crosses are sent back to our database and stored.

+
+
+
+
+

9.4 Crossing Experiment Detail Page

+

Information for crosses in the same crossing experiment is compiled in the crossing experiment detail page.

+

+

+

Each cross name, female parent, male parent, female plot and male plot has a link to its own detail page, which contains information specific to each one. +Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual.

+
+
+

9.5 Cross Detail Page

+

Information of each cross can also be viewed in its detail page.

+

+

+

This page allows you to update or edit crossing experimental information and add progenies related to that cross. +Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual.

+

+

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-downloads.html b/docs/managing-downloads.html new file mode 100644 index 0000000000..3a855eef2f --- /dev/null +++ b/docs/managing-downloads.html @@ -0,0 +1,463 @@ + + + + + + + Chapter 17 Managing Downloads | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 17 Managing Downloads

+

You can download phenotype, trial meta-data, pedigree, GBS genotype and GBS genotype QC files from the database to your computer by using “Lists”. To download, clicking on “Download” in the “Manage” menu.

+

+

For each category, you can select a list of accessions from your “Lists” to download their phenotypes, pedigree, GBS genotype, GBS genotype QC. In the case of downloading trial meta-data, you would provide a list of trials, while for downloading phenotype and GBS genotype QC, you can also use a list of trials or traits.

+

+

+

+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-field-trials.html b/docs/managing-field-trials.html new file mode 100644 index 0000000000..05b5b0dcb7 --- /dev/null +++ b/docs/managing-field-trials.html @@ -0,0 +1,964 @@ + + + + + + + Chapter 10 Managing Field Trials | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 10 Managing Field Trials

+

To view trial details on the database, click on the “Field Trials” link under the “manage” menu on the toolbar.

+

+

Clicking on the “Field Trials” link will bring you to the “Manage Trials” page. On this page, trials are organized according to their breeding programs. To access trial details, click on the + icon next to your breeding program.

+

+

Trials can be placed directly in their breeding program. Alternatively, they can be organized by using folders within each breeding program. Clicking on trial name will take you directly to the trial details page.

+

+
+

10.1 Trial Detail Page

+

The trial detail page displays important information about individual trials including breeding program, location, year, description of the trial, design, and any files associated with that trial.

+

+

Below the trial details you will find various menus for accessing and modifying trial data. There are sections for printing labels for your plots or plants, recording phenotypes, viewing your trial layout or design, viewing phenotypes for this trial, or conducting analyses.

+

+

The “transplanting date” field feature will only be displayed if it has a value. To add a transplanting date after creating a trial, change the show_transplanting_date parameter from 0 to 1 in the SGN config file. As a result, you will be able to add a date under the transplanting date field by clicking the “Edit Trial Details” on the trial detail page.

+

+
+
+

10.2 Adding Trials

+

Only users with the account status of “submitter” may create trials. To learn how to change your account status from “user” to “submitter” visit section 1.2.

+
+

10.2.1 Prerequisites

+
    +

  • To add a trial, all of your accessions should already exist in the database before you begin to design a trial. If you have accessions that are not in the database, see the instructions in Managing Accessions.

    • +

  • The breeding program and location for your trial should also exist in the database. If you need to add breeding program and/or location to the database, see the instructions in Managing Breeding Programs and Managing Locations respectively.

    • +
+

On the “Manage Trials” page, there are two methods to create trials: by selecting “Upload Existing Trial(s)” to create a trial or trials from a spreadsheet; or by selecting “Design New Trial” and entering the trial data by hand.

+

+
+
+

10.2.2 Adding a trial by using “Design New Trial” form

+
+

Step 1. Begin the “Design New Trial” workflow

+

Click on “Design New Trial” to begin.

+

+

The first step in this workflow is an introduction that looks like this:

+

+

Here it gives information about what is required for a trial, including that to create a new trial, you need to create a list of the accessions that you would like to use in the trial. Lists can be viewed, created, and modified with the “lists” tool at the upper right of the screen. For more information on lists, click here.

+
+
+

Step 2. Enter Trial Information

+

On this screen you need to enter basic information about the trial, such as breeding program and location(s). You must also select a design type, such as Complete Block Design. The design is important because it influences how your genotypes are distributed and randomized over the trial. You must first click validate before proceeding to the next step.

+

+
+
+

Step 3. Enter Design Information

+

On this screen you need to specify a list of accessions to use in the experiment. This list must be a valid list of accessions. You must also specify all required design information, such as number of replicates. In this case, the number of blocks must be given.

+

+
+
+

Step 4. Enter Trial Linkage Information (Optional)

+

This next sections allows you to associate this new trial with other field trials, crossing experiments, or genotyping plates already present in the database. This is optional, and can be completed at a later date from the trial detail page.

+

+
+
+

Step 5. Enter Field Map Information (Optional)

+

On this screen you can specify how the row and column numbers will be generated for the plots in the trial. The row and column number represent a relative position of the plot in the field. If you are not exactly sure of how you will plant the plots in the field or you have an irregular (non-rectangular) layout, you can skip this step for now. This information can be added on the Trial Detail Page once the trial is saved in the database in order to reflect exactly how the plots were planted in the field.

+

+
+
+

Step 6. Custom Plot Naming (Optional)

+

On this screen it is possible to change the format in which plot names will be generated for your trial. It is recommended to skip this step and just use the format generated by the database by default.

+

+
+
+

Step 7. Review Designed Trial

+

On this screen you can review the trial that the database has generated.

+

You will see a graphical representation of the trial. The numbers on the squares represent the plot_number of each plot and on mouse hover you can see further information about the plot.

+

+

You will also see a table representation of all the plots and their information. If you want to redo the randomization, you can click the “Redo Randomization” button.

+

+

At the bottom there is a brief summary of the trial followed by two buttons.

+

+
+
+

Step 8. Add Field Management Factors to your design (Optional)

+

You can add Field Management Factors by clicking “Add Field Management Factor(s) to Design”. Clicking this opens a dialog to specify your factor. You can name this to account for fertilizer, watering regime, inoculation, or anything else. The types of management factors available is customizable in the SGN configuration file. This is optional and can be added from the trial detail page afterwards.

+

+

Click “Continue” and a dialog will appear where you can specify plots or plants (if you added plants during trial creation) for which the factor was applied. There is a select all button as well.

+

+
+
+

Step 9. Saving new trial in the database

+

Once you are done reviewing the trial you can click “Confirm” to save the generated trial into the database. Once the trial has saved you will see the final completion screen:

+

+
+
+
+

10.2.3 Adding a trial from an uploaded file

+

If you already have trial design layout in a spreadsheet, you can add your trial into the database by using the “Upload Existing Trial(s)” button on the Manage Trials page.

+

+

Enter the information in the workflow to upload a trial from a spreadsheet.

+
+

Step 1:

+

The first step is to understand what the format of the trial upload is. It is important to understand that the field layout represents plots in the experiment. Each plot has a globally unique plot_name, a sequential plot_number that is unique in the trial (but not globally unique. e.g. 101, 102, 103 for three separate plots), an accession_name representing what genotype is planted in that plot, and a block_number representing design replication. Each plot can be thought of as having a row_number and a column_number representing the relative position of the plot in a grid (e.g. the top left plot is row 1 column 1 following by row 1 column 2). Each plot can be planted with an amount of seed from a seedlot, where the seedlot_name represents the specific seed packet that was used, and num_seed_per_plot and weight_gram_seed_per_plot represent amount that were transferred from the seedlot_name to the plot_name. Treatments can be applied onto plots using additional column names in your file, where a 1 represents if the treatment was applied to the plot and an empty cell means it was not applied.

+

+

The following page will allow you to pick a file for upload, including uploading multiple trials at once. On this page you can also inspect the file requirements for both single trial and multi-trial uploads.

+

+
+
Minimum File requirements
+
    +

  • All accession names in the file must exist in the database. See adding accessions for more information.

    • +

  • The uploaded file can be excel (.XLXS or .XLS), comma-separated values (.CSV), tab-separated values (.TSV) or semicolon-separated values (.SSV).

    • +

  • The first row (header) must contain the column names: plot_name accession_name plot_number block_number is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number

    • +

  • Only accession_name, plot_number, and block_number are required.

    • +
+

Minimal Example:

+ +++++++++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
plot_nameaccession_nameplot_numberblock_numberis_a_controlrep_numberrange_numberrow_numbercol_numberseedlot_namenum_seed_per_plotweight_gram_seed_per_plotentry_number
2018plot1my_accession110111
2018plot2my_accession22012
2018plot3my_accession21021
2018plot4my_accession120221
+

When uploading multiple trials, the requirements are the same, but with additional headers to differentiate trials in the same file: trial_name breeding_program location year transplanting_date design_type description trial_type trial_stock_type plot_width plot_length field_size planting_date harvest_date. Additionally, when uploading multiple trials, you may choose to get an email when the upload is complete (see section ?? )

+
+
+
File validation
+
    +
  • In case of errors in the uploaded file such as missing or invalid data, a window will appear listing the specific errors in the file that must be corrected before a successful upload.
    • +
+
+
+
Uploading a trial with Treatments
+
    +
  • You can upload a trial with treatments by adding additional column(s). The column header will be the treatment e.g. fertilizer, watering regime, inoculation, etc. and the values in these columns will be either 1 or empty, indicating that the treatment was applied to the plot or not.
    • +
+
+
+
+

Step 2:

+

Once you feel that your experiment field layout is in the right format, click on to the Next Step. You will see the following form which must be filled in completely:

+

+

The trial name must be globally unique in the database. Please try to follow standard naming conventions for your group.

+
+
+

Step 3:

+

Go to the next page where you can link this trial to other projects. Validate the form, and then you can click “Upload Trial”.

+

+
+
+

Step 4:

+

In the case where you have uploaded an experiment using accession_names that are not already present in the database, you will be taken to this screen. If the accession_names in your file are all already in the database, this step will be skipped. +The reason it is necessary for your accessions to be in the database before you can add a trial using them is that a single accession can be used among many trials and therefore must exist as a separate entity in the database; because of this it is also very important to be careful about adding wrongly duplicated accession_names into the database. From this screen it is possible to make a new list with the missing accession_names and then click “Add Accessions to the database” to immediately resolve the issue. Once all your accessions are in the database, click to move to the Next Step.

+

+
+
+

Step 5:

+

In the case where you have uploaded an experiment using seedlot_names that are not already present in the database, you will be taken to the next screen. If the seedlots in your file are all already in the database, this step will be skipped. +The reason it is necessary for your seedlots to be in the database before you can add a trial using them is that a single seedlot can be used among many trials and therefore must exist as a separate entity in the database. Once all your seedlots are in the database, click to move to the next step.

+
+
+

Step 6:

+

If there are any other errors with your file, such as if the plot_names are not globally unique in the database or your plot_numbers are not unique in your trial or row_number is not an integer or any other error, you will see the errors listed in the red box. It is up to you to correct these errors in your file. Simply open up the file you selected earlier in Excel and correct the issues and then save the file. Then you can click “Submit Trial” and it will resubmit it for you. You can continue to edit your file here and submit as many times as you need until it is accepted.

+

+
+
+

Completion screen

+

Whether you were lucky enough to submit your trial successfully on Step 2 or if you tried many times on Step 5, once your trial has been saved in the database you will see the following screen:

+

+
+
+
+

10.2.4 Multi-location trials

+

To add multi-location trials, simply select the multiple locations while using the ‘Add Trial’ form.

+

This will create a separate trial for each selected location, but they will share the same design and will be grouped in a single folder.

+

By default each trial design will have a fresh randomization, but if desired you may check the “Use same randomization for all locations” option.

+
+
+

Email alert for multiple trial design upload

+

When uploading multiple trials from a file, you have the option to receive email notifications by clicking the “Email Alert” checkbox. By default, the system will use the email address associated with your account, but you have the option to enter a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results.

+

+
+
+

10.2.5 Viewing Plot Layout and Trait HeatMap

+
+

10.2.5.1 Viewing plot layout

+

In the “Field Layout Tools and Phenotype Heatmap” section of a Trial Detail page, the trial physical layout is displayed by default. The relative position of the plots will be displayed based on the row and column positions given to the plots during the trial creation or upload steps. The plots are color-coded based on the plot’s rep and block numbers and whether or not it is used as a check. Hover the mouse over the plot to see details about a specific plot.

+

+
+
+

10.2.5.2 Viewing plot layout for multiple trials

+

If there is more than one trial grown in the same physical field, the trial layouts of all of the trials can be shown together if the trials share these properties:

+
    +
  • +Each trial has the same year +
    • +
  • +Each trial has the same location +
    • +
  • +The location type of the trials’ location is set to Field +
    • +
  • +The row and column positions of all of the plots (across the related trials) don’t overlap. For example, trial #1 starts at row 1 and trial #2 starts at row 10. +
    • +
+

When these conditions are met and you check the “Select Trials in Same Field” checkbox, the plots from all of the related trials will be displayed on the same field layout. The plots will be color-coded by trial. The planting order and harvest order downloads will include the plots from all of the displayed trials in the order in which the plots occur in the field.

+

+
+
+

Tracking plot images on fieldMap

+

Plot images can be seen on fieldMap if a plot is associated to any image.

+

+

To view plot image(s), click on a plot, a dialog will appear. In this dialog you will see a detailed overview of the plot, including what stock(s) are contained within it and their layout in the plot, if applicable. At the bottom of the dialog, you will see options to change the plot name or accession as well as view plot images.

+

+

On the resulting dialog, click on “View Plot Images.” To see more images if a plot has more that 2 images, click on See more images… Medium size of an image can be viewed by clicking on an image.

+

+
+
+

Viewing assayed trait heatmap

+

The phenotype heatmap can be viewed by selecting a specific assayed trait from the selectbox drop-down. Mousing over the plots highlights the plot in green and also displays the plot’s field information including the selected trait’s phenotype value.

+

+
+
+

Suppressing Plot Phenotype

+

Clicking on a plot on the heatmap would display a dialog that has a button for suppressing a plot phenotype value for a given trait. A suppressed plot value can be excluded during trial analysis and phenotype download.

+

+
+
+

10.2.5.3 Correcting spatial autocorrelation

+

For trials with spatial layout information and stored phenotypes, you can check and correct for spatial autocorrelation by clicking the “Calculate Spatial Correction” button above the field map. Doing so will open the spatial corrections dialog. For an in-depth review, visit the chapter on data analysis tools.

+

+
+
+
+

10.2.6 Adding additional information in the Trial Detail page

+

After you added a new trial to the database, you can edit trial details or add more information for that trial through the Trial Detail page.

+

#### Uploading Physical Trial Layout {-}

+

You can upload physical trial layout by clicking on the “Upload Spatial Layout” button on the Trial Detail page.

+

+

Please check file format carefully. You can find file format information by clicking on the “Spreadsheet format” on the “Upload Spatial Layout” window.

+

+

Spreadsheet format:

+

+
+

Physical Trial Layout File requirements

+
    +

  • All plot names in the file must exist in the database.

    • +

  • The uploaded file should be tab delimited (txt).

    • +

  • The first row (header) must contain the column names

    • +
+

Example:

+ + + + + + + + + + + + + + + + + + + + + + + + + +
plot_namerow_numbercol_number
plot111
plot212
plot313
+

Select the trial layout coordinates file that you want to upload for this trial, then click “OK” button to upload the file.

+

+

The following message is displayed after the coordinates are uploaded.

+

+ +
+
+

Downloading Field Map Spreadsheet

+

Field map spreadsheet can be downloaded if the trial has field coordinate (row and column numbers) uploaded for it plots. +To download, click on the Download FieldMap Layout link on the Trial Heatmap section.

+

+ +
+
+

Editing Physical Trial Layout

+

The “Usage Help” button contains information on how to edit physical trial layout.

+

+

+

There are two different options for editing trial layout:

+
    +

  • Replacing plot accession by clicking on the plot in the layout.

    • +

  • Replacing trial accession by using the “Edit Field Layout” button.

    • +
+

To edit a specific plot, click on that plot. Enter a new accession on the “Edit Plot Info” form, then click the “Replace Plot Accession” button.

+

+

To replace an accession (in every plot/plant of that accession), click on the “Edit Field Layout” button.

+

+

On the “Edit Field Layout” window, click the “Replace Accession” button.

+

+

Select any accession that you want to replace, enter your new accession, then click the “Replace Trial Accession” button.

+

+ +### Downloading the Trial Layout from the Trial Detail page

+

Click on “Download Layout” on the Trial Detail page under “Experimental Design”.

+

+

The trial layout includes all information regarding the observation units in the experiment. The observation units can be plots, plants, or subplots. The trial layout can include trial design information such as the block_number and rep_number. It can also include physical map information such as the row_number and col_number, if that information is available for the trial. +The trial layout also includes information regarding treatments that have been applied in the field. +Optionally, the layout can give information regarding accession’s global performance for a list of traits.

+

+
+
+
+

10.2.7 Adding Plant Entries To Your Trial

+

After you added a new trial to the database you can choose to add plant entries to your trial. Adding plant entries enables plant level phenotyping. It is generally better to enter data at the plant level into the database because it is always possible to calculate plot level phenotypes from the individual plant data.

+

Plant entries can be added to your trial in four ways: +(1) Automatically generated by the database. The only input required is the number of plants per plot. +(2) Uploaded in an XLS or XLSX file. This allows you to specifically name your plant entries. +(3) Uploaded using plant index number instead of name (the name is automatically generated) +(4) Uploaded using the number of plants per plot. This allows different numbers of plant per plot, as opposed to option 1.

+

Additionally, if a trial has had subplots added, each of these options can be used to add plants to subplots. Note that once subplots are added, plants cannot be added directly to plots, and once plants have been added, subplots cannot be created.

+

These options are available in the “Plant Entries” section on the Trial Detail page under “Experimental Design,” as shown below.

+

+
+

Automatically Generate Plant Entries

+

Clicking on “Add plant entries” opens the following dialog box. The only input required is the number of plants per plot. This will create plant entries that are named as a concatenation of the plot_name and the plant’s index number e.g. plot_name_plant_1. You may optionally add row and column data to give plants a spatial layout within each plot.

+

+
+
+

Upload Plant Entries

+

Alternatively, you can choose to upload an XLS or XLSX file that contains the names of the plant entries, the plant index numbers, or the number of plants per plot. Each option comes with a dialog that specifies the file formats.

+
+
+
+

10.2.8 Adding Tissue Sample Entries To Your Trial

+

Some trials require tissue samples to be collected from plants in a field trial. The database will generate these tissue sample identifiers for you and will maintain all relationships with the plant, plot, accession, etc. To begin, go to the Design section of a trial’s detail page and open the “tissue sample entries” section. +Please note that tissue samples are directly related to plants, therefore your trial requires plants before you can add tissue samples.

+

+

When you click on “Add tissue sample entries” you will see a dialog where you specify the number of tissue samples you require per plant. Once you have specified how many tissues samples, you can give specific words to distinguish samples, such as “root” or “stem”, as seen below.

+

+

Once you have added tissue sample entries they will appear in the design section of the trial as seen below.

+

+

Each tissue sample has a detail page where you can add information about the sample, such as if it is in transit or in storage somewhere.

+

+

The related stocks section near the bottom of this detail page displays the relationships between all stocks, including tissue samples.

+

+
+
+

10.2.9 Uploading GPS Coordinates For Plots

+

You can upload GPS coordinates for the plots in your trial. There is a link on the Trial Detail Page as shown below.

+

+

Clicking on this link will bring up the following dialog.

+

+

Here you can upload an XLS or XLSX file. To see information on the format of the file that should be uploaded, click on “Spreadsheet format”. This will bring up the following dialog.

+

+

This dialog tells you that the file must be XLS or XLSX and must contain: plot_name WGS84_bottom_left_x WGS84_bottom_left_y WGS84_bottom_right_x WGS84_bottom_right_y WGS84_top_right_x WGS84_top_right_y WGS84_top_left_x WGS84_top_left_y +The GPS coordinates should be WGS84 format and specify a four-pointed polygon around the plot.

+
+
+

10.2.10 Repetitive Measurements Section

+

If a trial includes repetitive traits or time-series values, you can effectively view and analyze these values through the Repetitive Measurements Section. Start by selecting the desired trait from the trait drop-down menu. Next, define the date range by either using the date-range picker or an interactive slider, which allows you to dynamically adjust the period you wish to examine. Once the date range is set, determine how to handle the repetitive measurements by choosing from various options such as First Value, Last Value, Averaged Value, Sum Values, or All Values. Choosing the “All Values” option enables an additional feature that visualizes the trend of the values over time, helping you identify patterns and trends within the data.

+

+
+
+

10.2.11 Uploading Additional Files To Trial

+

It may be of interest to you to upload additional documents, images, or recordings to your trial. To do this, scroll down to the “Uploaded Additional File” section on the trial detail page. From here you can view and download any of these additional files.

+

+

To upload an additional file, click on the “Upload Additional Files” link. A dialog will appear where you simply select your desired file. For information, you can click “Upload information” to see the following message.

+

+
+
+
+

10.3 Updating Trial Data

+

To updated the trial-level metadata (such as the planting date, design type, description, etc) of one or more existing trials, click the “Update Existing Trial(s)” button from the Manage > Field Trials page. This upload can also be used to rename trials or move trials to a different breeding program. In order to update a trial, you must be a curator or a submitter (that is associated with the breeding program of the trials).

+

+

Here you can upload a file that contains the new metadata for the existing trials in the database. The first column is labeled ‘trial_name’ and includes the name of the existing trial. Additional columns can be included for the metadata you want to update. Any columns not included in the file or values left blank will leave the existing metadata unchanged. The columns that can be included are:

+
    +
  • new_trial_name: A new name for the trial, must not already exist in the database
    • +
  • breeding_program: The name of breeding program that managed the trial, must exist in the database.
    • +
  • location: The name or abbreviation of the location where the trial was held, must exist in the database.
    • +
  • year: The year the trial was held.
    • +
  • transplanting_date: The transplanting_date of the trial was conducted. Date in YYYY-MM-DD format or ‘remove’ to remove the date
    • +
  • planting_date: Date of Planting in YYYY-MM-DD format or ‘remove’ to remove the date
    • +
  • harvest_date: Date of Harvest in YYYY-MM-DD format or ‘remove’ to remove the date
    • +
  • design_type: The shorthand for the design type, must exist in the database. Possible values include CRD: Completely Randomized Design, RCBD: Randomized Complete Block Design, RRC: Resolvable Row-Column, DRRC: Doubly-Resolvable Row-Column, ARC: Augmented Row-Column, Alpha: Alpha Lattice Design, Lattice: Lattice Design, Augmented: Augmented Design, MAD: Modified Augmented Design, greenhouse: undesigned Nursery/Greenhouse, splitplot: Split Plot, p-rep: Partially Replicated, Westcott: Westcott Design
    • +
  • description: Additional text with any other relevant information about the trial.
    • +
  • trial_type: The name of the trial type, must exist in the database. Possible values include Seedling Nursery, phenotyping_trial, Advanced Yield Trial, Preliminary Yield Trial, Uniform Yield Trial, Variety Release Trial, Clonal Evaluation, genetic_gain_trial, storage_trial, heterosis_trial, health_status_trial, grafting_trial, Screen House, Seed Multiplication, crossing_block_trial, Specialty Trial
    • +
  • plot_width: plot width in meters
    • +
  • plot_length: plot length in meters
    • +
  • field_size: field size in hectares
    • +
+
+
+

10.4 Deleting Trial Data

+

To delete a trial data, click on the “Delete trial data” section. There are links to delete traits, layout and trial entry data.

+

+

To delete assayed trait data, click on “Delete trait data” link. On the appeared dialog, confirm deletion by clicking on the “Select Traits For Deletion” button, then select one or more traits to delete from the trial.

+

+

+

To delete trial layout data, click on the “Delete layout data” link. Confirm deletion on the appeared dialog.

+

To Delete trial entry, click on “Delete trial entry” link. Confirm deletion on the appeared dialog.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-genotyping-plates.html b/docs/managing-genotyping-plates.html new file mode 100644 index 0000000000..b2975d7656 --- /dev/null +++ b/docs/managing-genotyping-plates.html @@ -0,0 +1,485 @@ + + + + + + + Chapter 11 Managing Genotyping Plates | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 11 Managing Genotyping Plates

+

Genotyping Plates represent the content of a genotyping plate sent to a genotyping facility (e.g. samples in specific wells). To streamline this process, it is possible to upload this information or let the database create a plate for you. +Once the genotyping plate is saved in the database it is then possible to export the information directly to genotyping facilities that are BrAPI compliant. The genotyping facility can then provide status information to us via BrAPI.

+

To begin go to Manage->Genotyping Plates.

+

+

Here the genotyping plates are divided by Breeding Program. These sections can be expanded by clicking on one.

+

+
+

11.1 Adding a New Genotyping Plate

+

To begin, click on “Add Genotyping Plate”. Notice that this form is split into three sections: “Plate Information”, “Well Information”, and “Confirm”. The first section is for defining information about the genotyping plate, such as a Plate identifier, plate format (96 well), etc. The second section is for defining the samples in the wells, such as sample names, sample concentrations, well position, etc. The final section is for Submitting the info.

+

All fields in the Plate Information section are required.

+

+

In the Well Information section you can choose to either 1) Upload an XLS or XLSX spreadsheet with your sample layout or 2) let the database create the sample layout.

+

+

If you choose to upload an XLS or XLSX spreadsheet, the Spreadsheet Template info requires the following:

+

+

In either case, the sample identifier is generally a concatenation of Plate name and well position, e.g. MyGenotypingTrial1_A01. +In either case, you need to provide a “source_observation_unit_name” for each sample. This can be a tissue sample name, a plant name, a plot name, or an accession name; however, in any case, the identifier must already exist in the database. This allows us to link the sample in the well to specific field trial plots, or, plants, or tissue_samples. If you only know which accession is in the well, you can use the accession name.

+

In the final Confirm section you can decide whether to submit this information to the genotyping facility you selected. This requires that the genotyping facility is BrAPI compliant to work.

+

+
+
+

11.2 Genotyping Plate Detail Page

+

If you open a specific genotyping plate, it will take you to the detail page. +Here you can see the Accessions used in the plate (if you created the trial and the source_observation_unit_names you used were plots, this will still work because we know the accession of the plot or plant or tissue sample).

+

+

Further down you can see a graphical representation of your plate with well positions. This can be 96 well or 384 well depending on your plate format.

+

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-high-dimensional-phenotyping-data.html b/docs/managing-high-dimensional-phenotyping-data.html new file mode 100644 index 0000000000..e2701115c9 --- /dev/null +++ b/docs/managing-high-dimensional-phenotyping-data.html @@ -0,0 +1,481 @@ + + + + + + + Chapter 14 Managing High Dimensional Phenotyping Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 14 Managing High Dimensional Phenotyping Data

+
+

14.1 Managing Transcriptomic Data

+
+

14.1.1 Uploading Transcriptomic Data

+

To upload transcriptomic data, go to the transcriptomics page by clicking “Transcriptomics” under the Manage tab.

+

+

Click the “Upload Transcriptomics Data” button to open the upload workflow dialog. On the second step “Samples”, you will be prompted to make sure that your samples are already in the database. You can refer to the “Managing Tissue Samples” chapter for instructions on how to create a sampling trial. After creating a sampling trial or confirming that your samples exist, move on to the “Protocol Info” step. Here you can select an existing protocol, or create a new protocol. To create a protocol, click the “Protocol Not Shown. Create a New Protocol” button. From here you can fill in information about your new protocol and when finished, click “Go to Next Step”.

+

+

You will need to have two csv files when uploading transcriptomics data. The first is the data matrix file, which should have these headers:

+

+

The names in the sample_name column should match the sample names you created in your sampling trial file. The header marked “transcript_name_columns” should be replaced with the transcript names you want to upload, each in its own column in the header and the respective expression value below. Example:

+

+

The second file is the transcript details file, which should have these headers:

+

+

Each row under the transcript_name column should contain each transcript name from your data matrix file. For example:

+

+

Once these two files are uploaded, click verify to check that they are formatted correctly and that the samples exist. If there are no errors, you can then click store to store your transcriptomics data.

+
+
+

14.1.2 Downloading Transcriptomics Data

+

To download your data, click the “Uploaded Transcriptomics Data” dropdown menu on the Transcriptomics page. Then click the “Your Uploaded Transcriptomic Data” dropdown. From here you will be able to see the transcriptomics data that you have uploaded. Click the “Download” button next to the data that you wish to download.

+

+ +
+
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-image-data.html b/docs/managing-image-data.html new file mode 100644 index 0000000000..b841d94a31 --- /dev/null +++ b/docs/managing-image-data.html @@ -0,0 +1,518 @@ + + + + + + + Chapter 21 Managing Image Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 21 Managing Image Data

+
+

21.1 Image-Phenotyping Dashboard

+
    +
  1. Upload raw image-captures in a compressed file (.zip) for orthophotomosaic assembly or upload previously stitched orthophotomosaic raster (.PNG, .JPG) imagery.
    2. +
  2. Dashboard shows all field trials and uploaded imaging events in collapsible sections.
    3. +
  3. Follow standard processes to manually create templates for assignment of plot-polygon images to the field experiment design.
    4. +
  4. All imagery is shown with the spectral category within collapsible sections. Figure shows NIR imagery.
    5. +
  5. Apply Fourier transform filtering, thresholding, and vegetation index masking. Plot-polygon images for all image processes are shown.
    6. +
  6. Extract and export phenotypic values from plot-polygon images for analyses and model training.
    7. +
+
+
+

21.2 Image Input

+

Clicking “Upload Imagery” will open the following dialog.

+

+

Raw-captures can be uploaded in a compressed (.zip) file so that they can be assembled into an orthophotomosaic. If orthophotomosaic assembly is not required, raster images (.PNG, .JPG) can be uploaded. Example data is given for raw Micasense RedEdge 5-band multispectral captures and for stitched orthophotomosaics.

+

+

To begin uploading images, a field trial must be selected. The field trial must already be saved in the database. For information about adding a field trial, please read the Field Trial documentation.

+

+

The image data is added to an imaging (drone run) event. Here you can select a previously saved imaging event or you can create a new one by defining a name, description, and date.

+

+

The uploaded data can be raw image-captures or complete raster images. Here you can select whether orthophotomosaic stitching is required.

+

+

In the case that orthophotomosaic stitching is required, select ‘yes’. On the next step you will see the following: +Upload a zipfile with the raw-captures. +When uploading Micasense RedEdge raw-captures, provide images of the Micasense calibration panels in a zipfile as well.

+

+

In the case that orthophotomosaic assembly is not required, simple upload the raster images. Select the number of image bands that will be uploaded e.g. for a five band multispectral camera, select 5.

+

+

In the caes that orthophotomosaic stitching is not required, select ‘no’. On the next step you will see the following:

+

+

Upload an image at each band with a unique name, description, and spectral type.

+
+
+

21.3 Standard Process

+

Once imagery is uploaded, it will appear on the dashboard under the field trial. Clicking the “Run Standard Process” button will begin extracting plot-polygon phenotypes from the imagery.

+

+

Clicking the button will open the following dialog.

+

+

Select a drone run band to use in this process. In the case of the Micasense 5 band multispectral camera there will be 5 bands shown here; select the NIR channel in this case because it has the highest contrast. In the case of standard color images, there will only be the RGB Color Image option here.

+

+

Rotate the image so that there the plots are oriented in a grid fashion. There can be a skew in the field layout, as seen in the following example.

+

+

Perform a rough cropping of the image by clicking on the four corners of the field. Cropping is important to remove any extraneous parts of the image.

+

+

This step shows a histogram of the cropped image. The standard process will magnitude threshold the top and low ends of the distribution.

+

+

In this step, the template for the plot polygons in the experimental field design are associated to the image. First, defined the number of rows and columns in the field experiment. Then click the four corners of the image, in respect to the top right, top left, botton left, and bottom right positions. Next click on “Draw Plot Polygon Template”. Review the template and clear/repeat the process until the template matches well. It is possible to “copy/paste” templates in the case where there are large breaks in the field design. Next, scroll down to the “assign Plot Polygons to Field Trial Entities” section. Select the location of Plot Number 1 as either “top left” or “top right” and whether the field design is serpentine or zigzag. Click on “Generate Assignments” and review that the names of the plots appear correctly in the overlay on the image. Finally, click “Finish and Save Polygons to Plots” when you have have confirmed the assignments.

+

+

Next, the dialog shows you that the standard process will be repeated for all uploaded image bands.

+

+

Next, choose which vegetation indices to apply.

+

+

Next, choose the phenotypic values to extract. You must define the time point for which the phenotype is; if the field trial has a planting date, the time point will automatically be populated as image date minus the planting date.

+

+

After completing the standard process, the job will continue in the background until it completes. You can check the status of the job from the dashboard.

+
+
+

21.4 Ground Control Points

+

Ground control points can be saved after an imaging event has undergone the standard process on orhomosaics. Ground control points can then be used across imaging events on the same field experiment in order to automate the entire standard process.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-locations.html b/docs/managing-locations.html new file mode 100644 index 0000000000..261bbdaa52 --- /dev/null +++ b/docs/managing-locations.html @@ -0,0 +1,463 @@ + + + + + + + Chapter 5 Managing Locations | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 5 Managing Locations

+

Field locations can be managed using the “Manage Locations” page. On this page, locations in the database are organized based on their breeding programs. Each location has a link to trials conducted in that location. To add a new location, click on the “Upload New Locations” button that links to the “Upload Locations” form.

+

+

The “Upload Locations” describes how to build a spreadsheet with location data for upload. Name, abbreviation, country code, country name, program, type, latitute, longitude, and elevation are all required. The NOAA station ID is optional. Link a spreadhsheet to the form and click “Upload” to add those locations to the database.

+

+

Alternatively, locations can be viewed and added via the map. Hover over an icon on the map to see the location details and trials linked to that location. Click on the map to open the new location dialog. Fill in the same information that would be used in the spreadsheet upload to add a new location.

+

+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-observation-variables.html b/docs/managing-observation-variables.html new file mode 100644 index 0000000000..ad57ea347a --- /dev/null +++ b/docs/managing-observation-variables.html @@ -0,0 +1,479 @@ + + + + + + + Chapter 20 Managing Observation Variables | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 20 Managing Observation Variables

+
+

20.1 Managing Observation Variables with Traits, Methods, and Scales

+

Observation variables are the identifiers used when collecting phenotypic data. An observation variable is composed of a trait, a method, and a scale. The trait describes the attribute being measured e.g. ‘Plant Height’. The method defines the protocol in which the trait was observed e.g. ‘Using a one meter long measuring stick’. The scale defines the units or dimensions for which the measurement was taken e.g. ‘Meters’.

+

Generally, observation variables are defined in ontologies that are predefined. We often use ontologies from cropontology.org. In this case, you will not be able to define your own observation variables directly; instead, you will need to contact us and we will add the observation variable for you.

+

For databases where the user has greater control, we have an interface to allow addition of observation variables, along with traits, methods, and scales. To begin, go to the Search->Traits page.

+

If the database you are on allows you to directly add observation variables, you will see the following button at the bottom of the page.

+

+

When you click the button, the following workflow will appear. You should be logged in or else it will not allow addition of the observation variable. +The workflow begins with an introduction.

+

+

On the next workflow step, you select the ontology that you want to insert the new observation variable into. You must also give a name and a definition for the new observation variable.

+

+

On the next workflow step, you select the trait ontology to use. Once you select a trait ontology, a select containing all the terms in the selected ontology will appear. You can either select a trait or if it does not exist in the select, you can create a new one by giving a name and a definition for the new trait.

+

+

On the next workflow step, you select the method ontology to use. Once you select a method ontology, a select containing all the terms in the selected ontology will appear. You can either select a method or if it does not exist in the select, you can create a new one by giving a name and a definition for the new method.

+

+

On the next workflow step, you select the scale ontology to use. Once you select a scale ontology, a select containing all the terms in the selected ontology will appear. You can either select a scale or if it does not exist in the select, you can create a new one by giving a name and a definition for the new scale. You can also define a format, minimum, maximum, categories, and default value for the new scale.

+

+

On the last page of the workflow, you confirm the submission.

+

+

Afterwards, you can use the newly created observation variable ontology term in your phenotyping.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-odk-data-collection.html b/docs/managing-odk-data-collection.html new file mode 100644 index 0000000000..22b39b6675 --- /dev/null +++ b/docs/managing-odk-data-collection.html @@ -0,0 +1,486 @@ + + + + + + + Chapter 18 Managing ODK Data Collection | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 18 Managing ODK Data Collection

+

To access this page go to Manage and then ODK Data Collection. +ODK is used for remotely collecting data on Android and IOS devices. We currently are working to support two ODK service providers, namely ONA and SMAP. We are using ONA to collect crossing information, including all lab activities following seed production. We are using SMAP for phenotypic data collection.

+
+

18.1 ONA Crossing Information

+
+

18.1.1 Managing ONA Crossing Information

+

+

To begin collecting data using the ONA ODK form you must first have a crossing plan in the form of a Cross Wishlist. To do this from this page, click the “Export Cross Wishlist to ONA” button. Please refer to the “Create Cross Wihlist” help section for more information. +It is possible to view the current available cross wishlists by clicking the “Export Cross Wishlist to ONA” button and then clicking “Available Cross Wishlists”.

+

Once your cross wishlist is available, you can use your mobile ODK application to record crosses being done realtime. You can also record all laboratory activities following seed extraction up to greenhouse plantlet hardening.

+

As you collect data using your mobile ODK application, your responses will be synchronized with our database. The “Schedule Import for Selected Form” section gives you options to perform the import daily or more frequently. It is also possible to initiate a data import from ONA at anytime by clicking “Import Crossing Data from Selected Form on ONA”.

+
+
+

18.1.2 Reviewing Plant Status

+

The mobile ODK application has options to collect information about the status of plants in the field, such as if they are flowering. Images for each plant can also be recorded. +The database will report this information here in a summary table that looks like the following. Notice that images are also transferred to the database.

+

+
+
+

18.1.3 Graphical Summary For Performed Crosses

+

There is a section to summarize activities done for each cross. In this table each row represents a single cross performed. All the activities that have been performed will be shown here, such as “first pollination” and “embryo rescue”. The scatter plot shown tracks seed numbers generated on the Y axis and date of activity on the X axis.

+

+
+
+

18.1.4 Summary Information For Performed Crosses

+

There is a secondary section to summarize what has been done across the entire Cross Wishlist. This tree structure shows all activities performed for a cross and shows how these crosses relate to the Cross Wishlist.

+

+ +
+
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-outliers-in-dataset.html b/docs/managing-outliers-in-dataset.html new file mode 100644 index 0000000000..c1e7e77cb1 --- /dev/null +++ b/docs/managing-outliers-in-dataset.html @@ -0,0 +1,505 @@ + + + + + + + Chapter 26 Managing Outliers in Dataset | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 26 Managing Outliers in Dataset

+
+

26.1 What is Outliers Functionality in Dataset ?

+

+

As in step The Search Wizard +we can create a dataset.

+

The dataset incorporates a feature to identify outlier points, which we may choose to exclude from a specific dataset. It’s important to note that these exclusions only apply at the dataset level, and no data is permanently removed from the database. Additionally, outlier categorization can be modified at any time, and these changes are visible to all other functionalities within the system.

+

Each dataset stores a wholly unique set of outlier points, completely independent of any other dataset in the database. Outliers are specifically designated for traits within datasets, exclusively encompassing phenotype data. If a particular dataset lacks traits as a part of wizard selection, this functionality is not available.

+

Each trait has its own set of defined outliers.

+
+
+

26.2 Accessing Trait Visualization

+

Once you’ve selected a specific trait, the web application provides access to a visualization of the data points associated with that trait.

+

+
+
+

26.3 Interpreting Visual Elements

+

Once you’ve selected a specific trait, the web application provides access to a visualization of the data points associated with that trait.

+
    +
  • Green Points: As per the legend, represent values for the selected trait that fall below the cut-off point set by the slider. (non-outliers)
    • +
  • Black Outlined Points: These data points are outlined with black borders, indicating that they are currently designated as outliers in the database.
    • +
  • Red Points: The red data points denote the cut-off points established by the slider for the allowable deviation value.
    • +
+

+
+
+

26.4 Choosing Cut-Off Values

+

You have two fundamental options for setting cut-off points:

+
    +
  • Median with MAD: This option involves using the median (middle value) along with the Mean Absolute Deviation (MAD) as a reference point for determining cut-off values.
    • +
  • Mean with Standard Deviation: Alternatively, you can choose to use the mean (average) in conjunction with the Standard Deviation to set cut-off points.
    • +
+
+
+

26.5 Setting Deviation Multiplier

+

The slider allows you to specify the deviation multiplier from a central point, which influences the cut-off values.

+
+
+

26.6 Utilizing Graph Controls

+

Beneath the graph, you’ll find four buttons, each serving a distinct function:

+
    +
  • Add selection to outliers: This button enables you to save the current cut-off points to the database for future reference.
    • +
  • Reset outliers for current trait: You can use this option to reset outliers for the selected trait.
    • +
  • Reset all outliers: This button allows you to reset outliers for the entire dataset.
    • +
  • Download Phenotype Table without outliers: You can download the phenotype data table in a comma-separated value format file, using this feature, with outliers excluded for selected dataset.
    • +
+

+

These tools and functions are designed to provide you with control and insights when working with data visualization and outliers.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-phenotypic-data.html b/docs/managing-phenotypic-data.html new file mode 100644 index 0000000000..c53d7ad3ca --- /dev/null +++ b/docs/managing-phenotypic-data.html @@ -0,0 +1,492 @@ + + + + + + + Chapter 13 Managing Phenotypic Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 13 Managing Phenotypic Data

+

To facilitate uploading process for phenotypic data, “Manage Phenotypic Data” page provides two options for uploading: Field Book Phenotype file in database format and phenotype file in Excel (.xls or .xlsx) file format. To access “Manage Phenotypic Data” page, clicking on “Phenotyping” in the “Manage” menu.

+

+
+

13.1 Uploading Fieldbook Phenotypes

+
+

13.1.1 Export Field Book Database File

+

The database upload of Field Book phenotype data relies on the “Database” format from the Field Book. Please make sure to export the “Database” format from the Field Book if you intend to upload the data using the Field Book Upload we describe below. If you prefer to use the “Table” format that the Field Book exports, you can modify this format to work with the Speadsheet Upload we describe below.

+
+
+

13.1.2 Upload Field Book Database File

+

To upload a Field Book Phenotype file in a database format, click the “Upload Fieldbook” link

+

+

The “Upload Fieldbook” link on this page and “Upload” link on the “Field Book Tools” page open the same dialogue. Please follow instructions for uploading phenotypic files on the ?? page.

+

+
+
+
+

13.2 Uploading Spreadsheet Phenotypes

+

To upload a phenotype file in an Excel (.xls or .xlsx) file format, click the “Upload Spreadsheet” link.

+

+

Please specify “Data Level” (Plots or Plants) and select the Excel file that you want to upload.

+

+
+

13.2.1 Generating Spreadsheet File

+

You can find more file format information by clicking on “Spreadsheet Format” link. Clicking on “Spreadsheet Format” will open the following dialog.

+

+

Clicking on “Create Phenotyping Spreadsheet” will bring up a dialog where you can indicate the trial(s) you are interested in and the trait list you are interested in. Clicking “Submit” will download the xlsx file onto your computer, where you can then fill in the phenotypes.

+

+
+
+

13.2.2 Uploading Spreadsheet File

+

To ensure that the file has a correct format for uploading, click on the “Verify” button. This will check the contents of the file and also perform quality checks on the values in the file. These checks include checking the trait definition for categorical values, minimum and maximum values, and data type checking. It will also check if there are already values uploaded for the given observation units and traits. If there are, there is an option to overwrite the existing values with the new values in your file. If the file is valid, only then can you click “Store” to store the information in the database.

+

+ +
+
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-phenotypic-image-data.html b/docs/managing-phenotypic-image-data.html new file mode 100644 index 0000000000..e72ef72193 --- /dev/null +++ b/docs/managing-phenotypic-image-data.html @@ -0,0 +1,475 @@ + + + + + + + Chapter 22 Managing Phenotypic Image Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 22 Managing Phenotypic Image Data

+
+

22.1 Uploading Image files

+

Clicking “Phenotyping” under Manage, and then “Upload Images” will open this dialog:

+

+

If uploading images directly, select the file format “Images”. A single image or multiple selected images can be uploaded using this file format. Each of these images must have been downloaded from the Fieldbook app, or have a filename that follows this structure: observationUnitName, traitname, number, timestamp joined by underscores. The filename can not include any underscores other than the ones separating each value. For example: 38873_branching_5_2025-07-28.jpg

+

If the images have been taken in the Fieldbook app, they will have EXIF data that includes their associated stocks and traits. This EXIF data will be automatically parsed and the associations will be made when the images are uploaded.

+

After selecting your images for upload, clicking verify will check if your filenames or image metadata is correct and corresponds to existing stocks in the database.

+

+

If the verification is successful, click store to save these images in the database.

+

If uploading images with associated phenotypic data, select the file format “Images with Associated Phenotypes”. A spreadsheet with the phenotypic values needs to be uploaded alongside a zipfile of images. The spreadsheet can be in .xls or .xlsx format, and the column headers need to be:

+

observationUnitName | observationVariableName | value | timestamp | image_name | person

+

The image_name column should match the file names of the associated images in the zipfile.

+
+
+

22.2 Uploading Images Directly to Stocks

+

In the Images section on a stock details page you can add a new image associated with that stock by clicking the “Add Image” button. This will associate any images uploaded through this dialog with the corresponding stock.

+

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-populations.html b/docs/managing-populations.html new file mode 100644 index 0000000000..8751997bd7 --- /dev/null +++ b/docs/managing-populations.html @@ -0,0 +1,465 @@ + + + + + + + Chapter 8 Managing Populations | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 8 Managing Populations

+

Populations are modeled as groups of accessions. This grouping can be useful in downstream analyses. To manage these populations go to Manage Accessions and scroll tp the bottom.

+

+

To add a new population click “Create Population”. The following dialog will appear where you choose a list of accessions and give a name to the new population. Please note it is also possible to create a population when you are uploading new accessions into the database.

+

+

Click on the plus (+) button next to Populations to see all the available populations. Click on a population name to see the accessions in the population.

+

+

From here you can delete accessions from a population as well as add new accessions to the population.

+

+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-seed-lots.html b/docs/managing-seed-lots.html new file mode 100644 index 0000000000..aaaa3308e7 --- /dev/null +++ b/docs/managing-seed-lots.html @@ -0,0 +1,588 @@ + + + + + + + Chapter 7 Managing Seed Lots | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 7 Managing Seed Lots

+

Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. The seed in seedlots can be from crosses or can be named accessions. Seedlots from crosses would represent seed harvested. Click Manage and then Seed Lots to begin.

+

+
+

7.1 Add New Seedlot(s)

+

To add a single new seedlot, click on “Add Seedlot”. This will bring up the following dialog where you enter information about where the seedlot exists, what accession or cross is contained in it, and how many seeds there are. A seedlot must contain either an accession or a cross, and not both. A seedlot must have a weight in grams or a seed count or both of these.

+

+

In the case where you have many seedlots to add to the database, you can upload an excel XLS or XLSX file instead. Click “Upload Seedlots” to see the following dialog.

+

+
+
+

7.2 Seedlot Transactions

+

Seedlots are capable of tracking where seeds came from, such as from crosses, and to where seeds go, such as to plots in the field. If you navigate to a seedlot detail page you will see the following.

+

+

On this page you see and can edit information regarding a single seedlot, such as its name and location. You will also see a table indicating all t he transactions that a seedlot has been involved in, such as if it was planted in a plot in the field. Transactions to field plots are created when adding or uploading a new trial or from a trial’s detail page. Clicking on “Add New Transaction” let you add a transaction from between this seedlot and another seedlot. This kind of transaction is useful for representing if you have distributed seed to different locations.

+

+
+
+

7.3 Seed Inventory

+

To inventory your seed: +1) Make sure your seedlots are in the database. Use “Add New Seedlot” to add a single seedlot or “Upload New Seedlots” to add many. +2) Make sure your seedlots are barcoded. You can print these barcodes from the database. +3) Use the “Inventory” Android Application to scan seedlot barcodes and record weight. Then use “Upload Inventory” to upload this info into database. If you prefer you can create your own CSV file and upload that, if you do not want to use the Inventory Application. +For more info about the “Inventory” Android Application go to Inventory.

+

Clicking the “Upload Inventory” button will bring the following dialog:

+

+

The CSV file that should contain your inventory should meet these Template requirements. The Seed Inventory Android Application exports this exact file.

+

+
+
+

7.4 Find Seedlots For a List of Accessions

+

A convenient tool for searching available seedlots for a list of accessions is available in the list tool. First open up your list of accessions. For help opening a list of accessions please see the List section help. There is a button called “See Available Seedlots”.

+

+

Once you click this, you will see the following table in a dialog. From here you can create a list of seedlots using the checkboxes and the input at the bottom.

+

+
+
+

7.5 Create a seedlot for an Accession or Cross

+

Complementary to what we saw above for creating seedlots from the “Manage Seedlots” page, it is possible to create a new seedlot from an accession’s detail page or from the cross detail page. On the accession detail page, this is visible in the “Related Stocks” section as seen below. The cross detail page has an identical section. Notice the link for creating a new seedlot, which streamlines adding the seedlot.

+

+
+
+

7.6 Add quality data to a seedlot

+

Quality information can be added to a seedlot in the quality field. This is also available as a column in the file upload format. It is recommended to use a controlled vocabulary, defined by the user, for the quality field. For example, good quality seed should be labelled “ok”, whereas other quality descriptors could be “moldy”, “insect damage”, or “low sprouting”, etc.

+
+
+

7.7 Seedlot Maintenance Events

+

For some crops, such as sugar kelp, a “seedlot” requires routine maintenance for the successful long-term storage of the seedlot. (For example, a Seedlot Maintenance Event for sugar kelp would be the routine change of the water that gametophytes are kept it). Breedbase can now store a record of these Seedlot Maintenance Events associated directly with existing Seedlots. Maintenance Events can be uploaded using a simple Excel template or recorded directly on the website.

+
+

7.7.1 Setup

+

Each Breedbase instance needs to be configured to support the storage of Seedlot Maintenance Events since each crop will have their own distinct set of maintenance events for their seedlots. To check if your Breedbase instance supports this feature, go to the Manage menu and select the Seed Lots page. Make sure you are logged in and look for the Seedlot Maintenance button near the top, next to the Create Seedlot(s) and Upload Inventory buttons. If you don’t see this button, contact the developer(s) supporting your Breedbase instance and ask if they can setup this feature.

+

+The location of the Seedlot Maintenance button on the Manage > Seed Lots page

+
+
+

7.7.2 Adding Events

+

Seedlot Maintenance Events can be added using two methods: 1) Uploading an Excel template or 2) Recording events directly on the website

+
+

Uploading Events with Excel Template

+

To bulk-upload a file of Seedlot Maintenance Events, first create an Excel (.xls or .xlsx) file with the following headers:

+
    +
  • seedlot - the name of the Seedlot to associate the event with (must exactly match an existing Seedlot in the database)
    • +
  • type - the name of the Seedlot Maintenance Event type (these vary between Breedbase instances, a list of supported event types is displayed on the upload page)
    • +
  • value - the value of the Seedlot Maintenance Event (these may be different for each event type and vary between Breedbase instances, a list of supported event values is displayed on the upload page)
    • +
  • notes - optional, additional notes/comments about the event
    • +
  • operator - the username of the Breedbase user that recorded the event
    • +
  • timestamp - the date/time the event was recorded, in ‘YYYY-MM-DD HH:MM:SS’ format
    • +
+

Once you have an Excel file with the events filled out, follow these steps to upload the events to the database:

+
    +
  1. Make sure you are logged in to your Breedbase instance
    2. +
  2. Go to the Manage > Seed Lots page
    3. +
  3. Select the Seedlot Maintenance button
    4. +
  4. Select the Upload Maintenance button
    5. +
  5. Choose your Excel (.xls or .xlsx) file to upload
    6. +
  6. Select the Upload button
    7. +
+

+The Seedlot Maintenance upload dialog, showing the supported event types and values (for sugar kelp)

+
+
+

Recording Events on Website

+

To add individual Seedlot Maintenance Events to the database in real time, as they’re being recorded, use the Record Maintenance page. Follow these steps to record Seedlot Maintenance Events:

+
    +
  1. Make sure you are logged in to your Breedbase instance
    2. +
  2. Go to the Manage > Seed Lots page
    3. +
  3. Select the Seedlot Maintenance button
    4. +
  4. Select the Record Maintenance button
    5. +
  5. Enter the Seedlot Name or scan a barcode that has the Seedlot Name encoded. Once entered, the box at the top of the page will display basic information about the Seedlot as well its recently recorded events.
    6. +
  6. Select or Enter the values of individual events
    7. +
  7. Optionally, notes button next to each event to add additional notes/comments about that specific event
    8. +
  8. Make sure the operator/username and timestamp are correct
    9. +
  9. Select the Submit button to add the recorded events to the database. NOTE: any events that remain selected as “Not Recorded” will not be submitted to the database.
    10. +
+

+The Seedlot Maintenance record page, as configured for sugar kelp

+
+
+
+

7.7.3 Displaying Events

+

Recently recorded Seedlot Maintenance Events are displayed in a table from the main Seedlot Maintenance page, as well as the detail page for individual Seedlots.

+

+Unfiltered table of recent Seedlot Maintenance events

+

The events displayed in these tables are sorted by timestamp, with the most recently recorded events displayed first. The displayed events can be filtered using any number of supported filter criteria, such as: +- seedlot names (as entered on the page or using an existing seedlot list), +- dates (on, on or before, before, on or after, and/or after the entered dates) +- event types +- event type values +- operator/username

+

Select the properties of the filter(s) you want to apply, then select the Add button next to the button to add the filter to the list of applied filters. Once you’re done adding filters, select the Filter button to search the database for the filtered events.

+

+A filtered table of Seedlot Maintenance events

+

The filtered events can be downloaded directly from the table using the Excel or CSV buttons at the top of the table. Or Seedlot Maintenance Events can be bulk-downloaded (this includes all events for a Seedlot) using a list of Seedlots from the main downloads page (see below).

+
+
+

7.7.4 Downloading Events

+

To bulk-download all events for a specific subset of Seedlots:

+
    +
  1. Create a list containing the Seelots you are interested in.
    2. +
  2. Go to the Download Using Lists page (Manage > Download)
    3. +
  3. Find the Download Seedlot Maintenance Events section
    4. +
  4. Select your list of Seedlots
    5. +
  5. Select the Download button to generate the download file
    6. +
+

The downloaded file will follow the same format as the upload template and will contain all recorded Seedlot Maintenance Events for each Seedlot in the list.

+
+
+
+

7.8 Deleting Seedlots

+

Seedlots can be deleted on the Manage Seedlots page (/breeders/seedlots) by search the seedlot and then clicking the X to delete one seedlot at a time. To delete a seedlot, the logged in user needs the required delete privileges on the seedlot. The seedlot also should not have any transactions associated with it (except for the initial transaction).

+

To delete seedlots in bulk, generate a list of type seedlot, for example, using the wizard. Open the section “Delete seedlots using a list” on the Manage Seedlots page and select the list. Seedlot deletion using a list is only available to user with curator status.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-sequence-metadata.html b/docs/managing-sequence-metadata.html new file mode 100644 index 0000000000..f9a04dc767 --- /dev/null +++ b/docs/managing-sequence-metadata.html @@ -0,0 +1,511 @@ + + + + + + + Chapter 25 Managing Sequence Metadata | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 25 Managing Sequence Metadata

+

+
+

25.1 What is Sequence Metadata?

+

Sequence Metadata is a feature that allows for the efficient storage and retrieval of sequence annotations for a specific region along a reference genome. The annotation data can contain a primary “score” value and any number of secondary key/value attribute data. For example, Sequence Metatadata can store MNase open chromatin scores for every 10 basepairs along the reference genome as well as genome-wide association study (GWAS) statistics, including the trait information associated with the result. This data can then be filtered by position and/or scores/attribute values and even cross-referenced with markers stored in the database.

+
+
+

25.2 Loading Sequence Metadata

+

Sequence Metadata can be loaded into the database using a gff3-formatted file. The following columns are used to load the data:

+
    +
  • #1 / seqid: The name of the database feature (ie chromosome) the metadata is associated with (The feature name must already exist as a feature in the database)
    • +
  • #4 / start: The metadata’s start position
    • +
  • #5 / end: The metadata’s end position
    • +
  • #6 / score: (optional) The primary score attribute of the metadata
    • +
  • #9 / attributes: (optional) Secondary key//value attributes to be saved with the score. These should be formatted using the gff3 standard (key1=value1;key2=value2). The attribute key cannot be either score, start, or end.
    • +
+

To upload the gff3 file:

+
    +
  1. Go to the Manage > Sequence Metadata page
    2. +
  2. Click the Upload Sequence Metadata button
    3. +
  3. On Step 2 of the Wizard, select the Type of data to be uploaded +
      +
    • This groups similar datasets together in the same Data Type category
      • +
    4. +
  4. On Step 3 of the Wizard, select an existing Protocol or create a new one +
      +
    • The Protocol is used to describe how the data was generated and define the score value and any secondary attributes. Adding the attributes (and their descriptions) to the Protocol will allow the Sequence Metadata queries to filter the data based on the value of one or more of these attributes. Attributes not defined in the Protocol will still be stored and displayed on retrieval, but will not be able to be used in a search filter.
      • +
    5. +
  5. Finally, select and upload your gff3 file to the database. The database will verify the format of the file before its contents are stored.
    6. +
+
+
+

25.3 Searching Sequence Metadata

+

To retrieve stored Sequence Metadata, go to the Search > Sequence Metadata page.

+ + +
+
+

25.4 Marker Integration

+

A table of Sequence Metadata annotations are embedded on the Marker/Variant detail page. The table will include any annotations that span the poisiton of the marker (for data of the same reference genome and species).

+
+
+

25.5 Sequence Metadata API

+

A publicly accessible RESTful API (Application Programming Interface) is available to query the database for Sequence Metadata directly from your programming environment (R, python, etc) to be used in analysis. The data is returned in a JSON format. Documentation for the API can be found on the Manage > Sequence Metadata page

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-spectral-data.html b/docs/managing-spectral-data.html new file mode 100644 index 0000000000..91ad4ab7a4 --- /dev/null +++ b/docs/managing-spectral-data.html @@ -0,0 +1,499 @@ + + + + + + + Chapter 24 Managing Spectral Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 24 Managing Spectral Data

+

Breedbase has implemented a flexible spectral data storage protocol that handles spectral data irrespective of the source spectrometer. Spectral data storage and analysis in Breedbase makes use of the R package waves for outlier identification, plotting, sample aggregation, and prediction model training.

+

+
+
+

24.1 Upload Spectral Data

+

Spectral data can be added as a CSV file that includes metadata in the leftmost columns followed by one column per spectral measurement to the right. Rows represent a single scan or sample, each with a unique ID that must match to a Breedbase observationUnitName. Future data transfer using BrAPI will allow for interoperability with data collection software.

+

To upload a spectral dataset, navigate to the ‘Manage NIRS Data’ page by selecting ‘NIRS’ in the ‘Manage’ menu and click the blue ‘Upload NIRS’ button. This will open an upload workflow. A link to the required file format and an example .csv file can be found by clicking in the light blue info box in this workflow. Another example of the file format is shown below.

+
    +
  • id: Optional identifier for each NIRS read. The id must be an integer.
    • +
  • sampling_id: Optional identifier for each sample. Strings are allowed.
    • +
  • sampling_date: Optional field. The format allowed is: YYYY-MM-DD.
    • +
  • observationunit_name: Required field that matches existing data in the database. It can be the plot name, subplots, plant name, or tissue sample, depending how your trial is designed.
    • +
  • device_id: Optional field to identify your device. Strings are allowed.
    • +
  • device_type: Required field. It is possible upload data for a single device type. They can be: SCiO, QST, Foss6500, BunchiN500, or LinkSquare.
    • +
  • comments: Optional field for general comments. +All other columns are required wavelengths. You can add how many columns you want upload – there is no limit.
    • +
+

+

+
+
+

24.2 Evaluate and Remove Outliers

+

Spectral calibration models can be heavily affected by the presence of outliers, whether they come from spectrometer spectral artifacts or user errors. Mahalanobis distance (Mahalanobis, 1936) is a measure of the distance between a single observation and a larger distribution and is commonly used in the identification of outliers in a multivariate space (Des Maesschalck et al, 2000). The FilterSpectra() function in the R package waves calculates the Mahalanobis distance of each observation in a given spectral matrix using the stats::mahalanobis() function. Observations are identified as outliers if the squared distance is greater than the 95th percentile of a \(\chi\)2-distribution with p degrees of freedom, where p is the number of columns (wavelengths) in the spectral matrix (Johnson and Wichern, 2007). In Breedbase, this procedure is applied on a per-dataset basis on upload and outliers are given binary tags “Outlier.”

+
+
+

24.3 Plot Spectra

+

After outlier identification, a plot is generated using the PlotSpectra() function in waves. This function uses the filtered spectra and ggplot2::ggplot() to create a line plot with outliers highlighted by color. A list of rows identified as outliers are shown beneath the plot. Plots are saved as .png files and linked to the original input datasets. Plot image files can be downloaded via the “Download Plot” button in the upload workflow.

+

+
+
+

24.4 Aggregate Spectra

+

To obtain a stable and reliable spectral profile, most spectrometer manufacturers recommend that multiple spectral scans are captured for each sample. While some spectrometers aggregate these scans internally, many do not, requiring the user to do so before analysis can take place. Breedbase handles these cases upon data upload following filtering steps by calling the AggregateSpectra() function from waves, saving the aggregated scans for future access through the search wizard feature. Scans are aggregated by sample mean (e.g. plot-level basis) according to the provided observationUnitName field. After aggregation, the user exits the upload workflow and the raw data file is saved in the upload archive.

+
+
+

24.5 References

+
    +
  • De Maesschalck, R., Jouan-Rimbaud, D., and Massart, D. L. (2000). The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18.
    • +
  • Johnson, R. A. & Wichern, D. W. (2007). Applied Multivariate Statistical Analysis (6th Edition). p 773.
    • +
  • Mahalanobis, P. C. (1936). On the generalized distance in statistics. National Institute of Science of India.
    • +
+

Analysis tool documentation

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-tissue-samples.html b/docs/managing-tissue-samples.html new file mode 100644 index 0000000000..928d6b53ca --- /dev/null +++ b/docs/managing-tissue-samples.html @@ -0,0 +1,487 @@ + + + + + + + Chapter 19 Managing Tissue Samples | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 19 Managing Tissue Samples

+

To access this page go to Manage and then Tissue Samples.

+
+

19.1 Tissue samples from field trials

+

A field trial contains plots planted with a specific accession. Each plot can contain many plants, which in turn can contain many tissue samples. +On the manage tissue sample page we can see the field trials that contain tissue samples already. We can choose to download the tissue sample layout as seen in the below picture.

+

+

If the field trial you want to collect tissue samples from is not in the above table, you can click the button highlighted below.

+

+

Once you have clicked this button, you will enter a workflow that begins with the following introduction.

+

+

Once you click next, you will need to select your trial.

+

+

Next, if your trial currently only has plot entries saved, you will be asked to enter how many plants are in each plot.

+

+

Finally you will be asked how many tissue samples you want for each plant. You can specify a string to include in the tissue sample name, such as leaf or root.

+

+

Afterwards you should see the following success message, indicating that the tissue samples are saved.

+

+
+
+

19.2 Genotyping Plate Tissue Samples (96 or 384 well plates)

+

A genotyping plate represents a 96 or 384 well plate. You can use the Coordinate Android application to create your plate layout, or you can upload your own Excel plate layout, or you can use the database to generate a plate layout. +Ideally, you will use tissue sample names originating from a field trial as the “source” for each well tissue sample, but you can also use plant names, plot names, or accession names.

+

From the manage tissue samples page, you can see the genotyping plates saved in the database. You can also download the layouts as shown below.

+

+

If you need to create a new genotyping plate, you can click the button shown below. This will guide you through a workflow for uploading or creating the new plate layout.

+

+

Genotyping vendors require you to send a plate layout during submission. You can download the plate layout as shown above, or you can go to a genotyping plate detail page to download the Intertek formatted file.

+

In the future you will be able to directly export your genotyping plate plate layout to vendors.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-user-roles.html b/docs/managing-user-roles.html new file mode 100644 index 0000000000..a8f228f642 --- /dev/null +++ b/docs/managing-user-roles.html @@ -0,0 +1,474 @@ + + + + + + + Chapter 3 Managing User Roles | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 3 Managing User Roles

+

+
+

3.1 What are User Roles?

+

Every user account in Breedbase has one or more associated “roles” that determine the authorizations (what the user is allowed to do) in the database. There are three fundamental roles, “curator”, “submitter”, and “user”, which determine basic read/write levels. The “curator” status can read and write everything in the database. The “submitter” status can add information and edit or delete previously submitted information. The “user” type can only read data. Additional roles represent the breeding programs, and are sometimes used to fine-tune write and edit capabilities, as it necessary for multiple users in a breeding program to edit each other’s data.

+
+
+

3.2 The Manage User Roles page

+

In the “Manage” menu, select the item “User Roles”. This will show the current users in the database with their associated roles. If you are logged in as a curator, the table will show system roles as well as breeding program roles; if you are logged in as a submitter or user, it will show breeding program membership.

+

If logged in as a “curator”, the roles can be added or deleted.

+
    +
  • To delete a role, click on the X in the role name. A confirm dialog will be displayed to prevent accidental deletion.
    • +
  • To add a role, click on the plus sign next to the roles. A dialog will pop up with a list of roles. Select the desired role and click “Submit”.
    • +
  • The new role should be displayed next to the user immediately.
    • +
  • Role deletions and additions will be effective immediately.
    • +
+

It is recommended that few users be given the “curator” privileges to avoid confusion over data ownership and accidental data overwriting and deletion.

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/managing-vcf-data.html b/docs/managing-vcf-data.html new file mode 100644 index 0000000000..1fbbb8900d --- /dev/null +++ b/docs/managing-vcf-data.html @@ -0,0 +1,499 @@ + + + + + + + Chapter 23 Managing VCF Data | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Chapter 23 Managing VCF Data

+
+

23.1 Uploading VCF Data

+

Genotyping data in VCF can be loaded from the web-interface. Breedbase can store any genotypic variants from a VCF, allowing for polyploids, structural variants, etc. without problems.

+

To begin go to Manage->Genotyping Plates and click the button seen below: +Note that you do not need to have genotyping plates uploaded to upload VCF data; you may upload genotyping data to accessions or you can upload genotyping data for tissue samples in genotyping plates.

+

+

The workflow begins with an intro:

+

+

On the following step in the workflow, a genotyping project is defined or selected. A genotyping project is a high-level entity for grouping several genotyping events. It is defined with a name, description, name, breeding program, and genotyping facility (IGD, Intertek, etc.).

+

+

The following step is to define or select a genotyping protocol. A genotyping protocol represents the set of markers being called against a specific reference genome. A genotyping protocol is defined with a name, description, reference genome name, species name, and a location of data generation. Note in the picture that you can select whether the samples in your file are accessions or tissue samples in the database; tissue samples are for when a genotyping plate is stored in the database. There is an option to parse the sample names for appended sequencing numbers from IGD, where the sample names are like “accession:igdnumber”.

+

+

The final step is to select the VCF from your computer and upload it. The web interface can be used to upload files arbitrarily large; it is a NGINX configuration to set this size.

+

+
+
+

23.2 Searching and Downloading VCF Data

+

The Search Wizard is the primary means of querying data in the database. Go to Search->Wizard to begin.

+

Once genotyping protocols are stored, select Genotyping Protocols from the first dropdown menu. Then if you select one or more and select Accessions from the second dropdown menu, you will see the accessions for which genotypes were stored. As seen in the following picture, there is a section for filtering genotypes by chromosome, start position, and end position. Genotypes can be downloaded in VCF or DosageMatrix formats.

+

+

Using the “Default genotyping protocol” which is configured in a system, you can query over field phenotypic evaluations before downloading genotypes and phenotypes.

+

+
+
+

23.3 Searching Protocols

+

Genotyping protocols can be search by going to Search->Genotyping Protocols. To download genotypes accessions must be selected, though any combination of search criteria can be used to filter and select those accessions. If a genotyping protocol is not selected, then the default genotyping protocol set in the configuration will be used. Genotyping protocols can also be selected in the wizard.

+

+

The genotyping download menu on the Search Wizard presents options for filtering by chromosome, start position, and end position. Genotypes can be downloaded in VCF of Dosage Matrix formats. The genomic relationship matrix (GRM) can be downloaded for the selected accessions in a tab-delimited matrix format or in a three-column format that is useful in Asreml. Genotypes can be computed from the parents in the pedigree if those parents are genotyped by clicking on the “compute from parents” checkbox. Additionally, the GRM can be computed using genotypes of parents in the pedigree if the “compute from parents” checkbox is selected.

+

+

As is described elsewhere, the Search Wizard presents a way to filter phenotypic values by minimum and maximum values, and allow for download in CSV and Excel formats.

+

+
+
+

23.4 Detail Pages and Deletion

+

The genotyping protocol detail page will show all information about the protocol such as the reference genome used, the header information lines in the uploaded VCF file, the markers involved, and the samples genotyped.

+

The markers section will show all markers used and their annotations, such as position, chromosome, alternate allele, reference allele, marker format, etc.

+

+

The samples section will show all samples genotyped. Notice the Download links in the table which can be used to easily get the VCF file results for each genotyped samples with all markers in the genotyping protocol. For getting mulitple samples at once, use the Search Wizard as discussed above.

+

+

The genotyping protocol and all associated genotyping data can be deleted from the genotyping protocol page.

+

+ +
+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/r_markdown_docs/BreedbaseManual.log b/docs/r_markdown_docs/BreedbaseManual.log new file mode 100644 index 0000000000..ac53c7edd3 --- /dev/null +++ b/docs/r_markdown_docs/BreedbaseManual.log @@ -0,0 +1,2737 @@ +This is XeTeX, Version 3.141592653-2.6-0.999995 (TeX Live 2023) (preloaded format=xelatex 2023.10.29) 8 JAN 2026 17:45 +entering extended mode + restricted \write18 enabled. + %&-line parsing enabled. +**BreedbaseManual.tex +(./BreedbaseManual.tex +LaTeX2e <2022-11-01> patch level 1 +L3 programming layer <2023-02-22> (/usr/share/texlive/texmf-dist/tex/latex/base +/book.cls +Document Class: book 2022/07/02 v1.4n Standard LaTeX document class +(/usr/share/texlive/texmf-dist/tex/latex/base/bk12.clo +File: bk12.clo 2022/07/02 v1.4n Standard LaTeX file (size option) +) +\c@part=\count181 +\c@chapter=\count182 +\c@section=\count183 +\c@subsection=\count184 +\c@subsubsection=\count185 +\c@paragraph=\count186 +\c@subparagraph=\count187 +\c@figure=\count188 +\c@table=\count189 +\abovecaptionskip=\skip48 +\belowcaptionskip=\skip49 +\bibindent=\dimen140 +) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsmath.sty +Package: amsmath 2022/04/08 v2.17n AMS math features +\@mathmargin=\skip50 +For additional information on amsmath, use the `?' option. +(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amstext.sty +Package: amstext 2021/08/26 v2.01 AMS text +(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsgen.sty +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks16 +\ex@=\dimen141 +)) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsbsy.sty +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen142 +) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsopn.sty +Package: amsopn 2022/04/08 v2.04 operator names +) +\inf@bad=\count190 +LaTeX Info: Redefining \frac on input line 234. +\uproot@=\count191 +\leftroot@=\count192 +LaTeX Info: Redefining \overline on input line 399. +LaTeX Info: Redefining \colon on input line 410. +\classnum@=\count193 +\DOTSCASE@=\count194 +LaTeX Info: Redefining \ldots on input line 496. +LaTeX Info: Redefining \dots on input line 499. +LaTeX Info: Redefining \cdots on input line 620. +\Mathstrutbox@=\box51 +\strutbox@=\box52 +LaTeX Info: Redefining \big on input line 722. +LaTeX Info: Redefining \Big on input line 723. +LaTeX Info: Redefining \bigg on input line 724. +LaTeX Info: Redefining \Bigg on input line 725. +\big@size=\dimen143 +LaTeX Font Info: Redeclaring font encoding OML on input line 743. +LaTeX Font Info: Redeclaring font encoding OMS on input line 744. +\macc@depth=\count195 +LaTeX Info: Redefining \bmod on input line 905. +LaTeX Info: Redefining \pmod on input line 910. +LaTeX Info: Redefining \smash on input line 940. +LaTeX Info: Redefining \relbar on input line 970. +LaTeX Info: Redefining \Relbar on input line 971. +\c@MaxMatrixCols=\count196 +\dotsspace@=\muskip16 +\c@parentequation=\count197 +\dspbrk@lvl=\count198 +\tag@help=\toks17 +\row@=\count199 +\column@=\count266 +\maxfields@=\count267 +\andhelp@=\toks18 +\eqnshift@=\dimen144 +\alignsep@=\dimen145 +\tagshift@=\dimen146 +\tagwidth@=\dimen147 +\totwidth@=\dimen148 +\lineht@=\dimen149 +\@envbody=\toks19 +\multlinegap=\skip51 +\multlinetaggap=\skip52 +\mathdisplay@stack=\toks20 +LaTeX Info: Redefining \[ on input line 2953. +LaTeX Info: Redefining \] on input line 2954. +) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amssymb.sty +Package: amssymb 2013/01/14 v3.01 AMS font symbols +(/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amsfonts.sty +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +)) (/usr/share/texlive/texmf-dist/tex/generic/iftex/iftex.sty +Package: iftex 2022/02/03 v1.0f TeX engine tests +) (/usr/share/texlive/texmf-dist/tex/latex/unicode-math/unicode-math.sty (/usr/ +share/texlive/texmf-dist/tex/latex/l3kernel/expl3.sty +Package: expl3 2023-02-22 L3 programming layer (loader) +(/usr/share/texlive/texmf-dist/tex/latex/l3backend/l3backend-xetex.def +File: l3backend-xetex.def 2023-01-16 L3 backend support: XeTeX +\g__graphics_track_int=\count268 +\l__pdf_internal_box=\box53 +\g__pdf_backend_object_int=\count269 +\g__pdf_backend_annotation_int=\count270 +\g__pdf_backend_link_int=\count271 +)) +Package: unicode-math 2020/01/31 v0.8q Unicode maths in XeLaTeX and LuaLaTeX +(/usr/share/texlive/texmf-dist/tex/latex/unicode-math/unicode-math-xetex.sty +Package: unicode-math-xetex 2020/01/31 v0.8q Unicode maths in XeLaTeX and LuaLa +TeX +(/usr/share/texlive/texmf-dist/tex/latex/l3packages/xparse/xparse.sty +Package: xparse 2023-02-02 L3 Experimental document command parser +) (/usr/share/texlive/texmf-dist/tex/latex/l3packages/l3keys2e/l3keys2e.sty +Package: l3keys2e 2023-02-02 LaTeX2e option processing using LaTeX3 keys +) (/usr/share/texlive/texmf-dist/tex/latex/fontspec/fontspec.sty +Package: fontspec 2022/01/15 v2.8a Font selection for XeLaTeX and LuaLaTeX +(/usr/share/texlive/texmf-dist/tex/latex/fontspec/fontspec-xetex.sty +Package: fontspec-xetex 2022/01/15 v2.8a Font selection for XeLaTeX and LuaLaTe +X +\l__fontspec_script_int=\count272 +\l__fontspec_language_int=\count273 +\l__fontspec_strnum_int=\count274 +\l__fontspec_tmp_int=\count275 +\l__fontspec_tmpa_int=\count276 +\l__fontspec_tmpb_int=\count277 +\l__fontspec_tmpc_int=\count278 +\l__fontspec_em_int=\count279 +\l__fontspec_emdef_int=\count280 +\l__fontspec_strong_int=\count281 +\l__fontspec_strongdef_int=\count282 +\l__fontspec_tmpa_dim=\dimen150 +\l__fontspec_tmpb_dim=\dimen151 +\l__fontspec_tmpc_dim=\dimen152 +(/usr/share/texlive/texmf-dist/tex/latex/base/fontenc.sty +Package: fontenc 2021/04/29 v2.0v Standard LaTeX package +) (/usr/share/texlive/texmf-dist/tex/latex/fontspec/fontspec.cfg))) (/usr/share +/texlive/texmf-dist/tex/latex/base/fix-cm.sty +Package: fix-cm 2020/11/24 v1.1t fixes to LaTeX +(/usr/share/texlive/texmf-dist/tex/latex/base/ts1enc.def +File: ts1enc.def 2001/06/05 v3.0e (jk/car/fm) Standard LaTeX file +LaTeX Font Info: Redeclaring font encoding TS1 on input line 47. +)) +\g__um_fam_int=\count283 +\g__um_fonts_used_int=\count284 +\l__um_primecount_int=\count285 +\g__um_primekern_muskip=\muskip17 +(/usr/share/texlive/texmf-dist/tex/latex/unicode-math/unicode-math-table.tex))) +(/usr/share/texlive/texmf-dist/tex/latex/lm/lmodern.sty +Package: lmodern 2015/05/01 v1.6.1 Latin Modern Fonts +LaTeX Font Info: Overwriting symbol font `operators' in version `normal' +(Font) OT1/cmr/m/n --> OT1/lmr/m/n on input line 22. +LaTeX Font Info: Overwriting symbol font `letters' in version `normal' +(Font) OML/cmm/m/it --> OML/lmm/m/it on input line 23. +LaTeX Font Info: Overwriting symbol font `symbols' in version `normal' +(Font) OMS/cmsy/m/n --> OMS/lmsy/m/n on input line 24. +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `normal' +(Font) OMX/cmex/m/n --> OMX/lmex/m/n on input line 25. +LaTeX Font Info: Overwriting symbol font `operators' in version `bold' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 26. +LaTeX Font Info: Overwriting symbol font `letters' in version `bold' +(Font) OML/cmm/b/it --> OML/lmm/b/it on input line 27. +LaTeX Font Info: Overwriting symbol font `symbols' in version `bold' +(Font) OMS/cmsy/b/n --> OMS/lmsy/b/n on input line 28. +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `bold' +(Font) OMX/cmex/m/n --> OMX/lmex/m/n on input line 29. +LaTeX Font Info: Overwriting math alphabet `\mathbf' in version `normal' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 31. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `normal' +(Font) OT1/cmss/m/n --> OT1/lmss/m/n on input line 32. +LaTeX Font Info: Overwriting math alphabet `\mathit' in version `normal' +(Font) OT1/cmr/m/it --> OT1/lmr/m/it on input line 33. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `normal' +(Font) OT1/cmtt/m/n --> OT1/lmtt/m/n on input line 34. +LaTeX Font Info: Overwriting math alphabet `\mathbf' in version `bold' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 35. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `bold' +(Font) OT1/cmss/bx/n --> OT1/lmss/bx/n on input line 36. +LaTeX Font Info: Overwriting math alphabet `\mathit' in version `bold' +(Font) OT1/cmr/bx/it --> OT1/lmr/bx/it on input line 37. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `bold' +(Font) OT1/cmtt/m/n --> OT1/lmtt/m/n on input line 38. +) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.sty +Package: microtype 2023/03/13 v3.1a Micro-typographical refinements (RS) +(/usr/share/texlive/texmf-dist/tex/latex/graphics/keyval.sty +Package: keyval 2022/05/29 v1.15 key=value parser (DPC) +\KV@toks@=\toks21 +) (/usr/share/texlive/texmf-dist/tex/latex/etoolbox/etoolbox.sty +Package: etoolbox 2020/10/05 v2.5k e-TeX tools for LaTeX (JAW) +\etb@tempcnta=\count286 +) +\MT@toks=\toks22 +\MT@tempbox=\box54 +\MT@count=\count287 +LaTeX Info: Redefining \noprotrusionifhmode on input line 1059. +LaTeX Info: Redefining \leftprotrusion on input line 1060. +\MT@prot@toks=\toks23 +LaTeX Info: Redefining \rightprotrusion on input line 1078. +LaTeX Info: Redefining \textls on input line 1368. +\MT@outer@kern=\dimen153 +LaTeX Info: Redefining \textmicrotypecontext on input line 1988. +\MT@listname@count=\count288 +(/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype-xetex.def +File: microtype-xetex.def 2023/03/13 v3.1a Definitions specific to xetex (RS) +LaTeX Info: Redefining \lsstyle on input line 238. +) +Package microtype Info: Loading configuration file microtype.cfg. +(/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.cfg +File: microtype.cfg 2023/03/13 v3.1a microtype main configuration file (RS) +)) (/usr/share/texlive/texmf-dist/tex/latex/parskip/parskip.sty +Package: parskip 2021-03-14 v2.0h non-zero parskip adjustments +(/usr/share/texlive/texmf-dist/tex/latex/kvoptions/kvoptions.sty +Package: kvoptions 2022-06-15 v3.15 Key value format for package options (HO) +(/usr/share/texlive/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty +Package: ltxcmds 2020-05-10 v1.25 LaTeX kernel commands for general use (HO) +) (/usr/share/texlive/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty +Package: kvsetkeys 2022-10-05 v1.19 Key value parser (HO) +))) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/xcolor.sty +Package: xcolor 2022/06/12 v2.14 LaTeX color extensions (UK) +(/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/color.cfg +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package xcolor Info: Driver file: xetex.def on input line 227. +(/usr/share/texlive/texmf-dist/tex/latex/graphics-def/xetex.def +File: xetex.def 2022/09/22 v5.0n Graphics/color driver for xetex +) (/usr/share/texlive/texmf-dist/tex/latex/graphics/mathcolor.ltx) +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1353. +Package xcolor Info: Model `RGB' extended on input line 1369. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1371. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1372. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1373. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1374. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1375. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1376. +(/usr/share/texlive/texmf-dist/tex/latex/graphics/dvipsnam.def +File: dvipsnam.def 2016/06/17 v3.0m Driver-dependent file (DPC,SPQR) +) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/svgnam.def +File: svgnam.def 2022/06/12 v2.14 Predefined colors according to SVG 1.1 (UK) +) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/x11nam.def +File: x11nam.def 2022/06/12 v2.14 Predefined colors according to Unix/X11 (UK) +)) (/usr/share/texlive/texmf-dist/tex/latex/tools/longtable.sty +Package: longtable 2021-09-01 v4.17 Multi-page Table package (DPC) +\LTleft=\skip53 +\LTright=\skip54 +\LTpre=\skip55 +\LTpost=\skip56 +\LTchunksize=\count289 +\LTcapwidth=\dimen154 +\LT@head=\box55 +\LT@firsthead=\box56 +\LT@foot=\box57 +\LT@lastfoot=\box58 +\LT@gbox=\box59 +\LT@cols=\count290 +\LT@rows=\count291 +\c@LT@tables=\count292 +\c@LT@chunks=\count293 +\LT@p@ftn=\toks24 +) (/usr/share/texlive/texmf-dist/tex/latex/booktabs/booktabs.sty +Package: booktabs 2020/01/12 v1.61803398 Publication quality tables +\heavyrulewidth=\dimen155 +\lightrulewidth=\dimen156 +\cmidrulewidth=\dimen157 +\belowrulesep=\dimen158 +\belowbottomsep=\dimen159 +\aboverulesep=\dimen160 +\abovetopsep=\dimen161 +\cmidrulesep=\dimen162 +\cmidrulekern=\dimen163 +\defaultaddspace=\dimen164 +\@cmidla=\count294 +\@cmidlb=\count295 +\@aboverulesep=\dimen165 +\@belowrulesep=\dimen166 +\@thisruleclass=\count296 +\@lastruleclass=\count297 +\@thisrulewidth=\dimen167 +) (/usr/share/texlive/texmf-dist/tex/latex/tools/array.sty +Package: array 2022/09/04 v2.5g Tabular extension package (FMi) +\col@sep=\dimen168 +\ar@mcellbox=\box60 +\extrarowheight=\dimen169 +\NC@list=\toks25 +\extratabsurround=\skip57 +\backup@length=\skip58 +\ar@cellbox=\box61 +) (/usr/share/texlive/texmf-dist/tex/latex/tools/calc.sty +Package: calc 2017/05/25 v4.3 Infix arithmetic (KKT,FJ) +\calc@Acount=\count298 +\calc@Bcount=\count299 +\calc@Adimen=\dimen170 +\calc@Bdimen=\dimen171 +\calc@Askip=\skip59 +\calc@Bskip=\skip60 +LaTeX Info: Redefining \setlength on input line 80. +LaTeX Info: Redefining \addtolength on input line 81. +\calc@Ccount=\count300 +\calc@Cskip=\skip61 +) (/usr/share/texlive/texmf-dist/tex/latex/footnotehyper/footnotehyper.sty +Package: footnotehyper 2021/08/13 v1.1e hyperref aware footnote.sty (JFB) +\FNH@notes=\box62 +\FNH@width=\dimen172 +\FNH@toks=\toks26 +) (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 2021/09/16 v1.2d Enhanced LaTeX Graphics (DPC,SPQR) +(/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2022/03/10 v1.4e Standard LaTeX Graphics (DPC,SPQR) +(/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 2021/08/11 v1.11 sin cos tan (DPC) +) (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: xetex.def on input line 107. +) +\Gin@req@height=\dimen173 +\Gin@req@width=\dimen174 +) (/usr/share/texlive/texmf-dist/tex/latex/natbib/natbib.sty +Package: natbib 2010/09/13 8.31b (PWD, AO) +\bibhang=\skip62 +\bibsep=\skip63 +LaTeX Info: Redefining \cite on input line 694. +\c@NAT@ctr=\count301 +) (/usr/share/texlive/texmf-dist/tex/latex/bookmark/bookmark.sty +Package: bookmark 2020-11-06 v1.29 PDF bookmarks (HO) +(/usr/share/texlive/texmf-dist/tex/latex/hyperref/hyperref.sty +Package: hyperref 2023-02-07 v7.00v Hypertext links for LaTeX +(/usr/share/texlive/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty +Package: pdftexcmds 2020-06-27 v0.33 Utility functions of pdfTeX for LuaTeX (HO +) +(/usr/share/texlive/texmf-dist/tex/generic/infwarerr/infwarerr.sty +Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO) +) +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode not found. +) (/usr/share/texlive/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty +Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO) +) (/usr/share/texlive/texmf-dist/tex/generic/pdfescape/pdfescape.sty +Package: pdfescape 2019/12/09 v1.15 Implements pdfTeX's escape features (HO) +) (/usr/share/texlive/texmf-dist/tex/latex/hycolor/hycolor.sty +Package: hycolor 2020-01-27 v1.10 Color options for hyperref/bookmark (HO) +) (/usr/share/texlive/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty +Package: letltxmacro 2019/12/03 v1.6 Let assignment for LaTeX macros (HO) +) (/usr/share/texlive/texmf-dist/tex/latex/auxhook/auxhook.sty +Package: auxhook 2019-12-17 v1.6 Hooks for auxiliary files (HO) +) (/usr/share/texlive/texmf-dist/tex/latex/hyperref/nameref.sty +Package: nameref 2022-05-17 v2.50 Cross-referencing by name of section +(/usr/share/texlive/texmf-dist/tex/latex/refcount/refcount.sty +Package: refcount 2019/12/15 v3.6 Data extraction from label references (HO) +) (/usr/share/texlive/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty +Package: gettitlestring 2019/12/15 v1.6 Cleanup title references (HO) +) +\c@section@level=\count302 +) +\@linkdim=\dimen175 +\Hy@linkcounter=\count303 +\Hy@pagecounter=\count304 +(/usr/share/texlive/texmf-dist/tex/latex/hyperref/pd1enc.def +File: pd1enc.def 2023-02-07 v7.00v Hyperref: PDFDocEncoding definition (HO) +) (/usr/share/texlive/texmf-dist/tex/generic/intcalc/intcalc.sty +Package: intcalc 2019/12/15 v1.3 Expandable calculations with integers (HO) +) (/usr/share/texlive/texmf-dist/tex/generic/etexcmds/etexcmds.sty +Package: etexcmds 2019/12/15 v1.7 Avoid name clashes with e-TeX commands (HO) +) +\Hy@SavedSpaceFactor=\count305 +(/usr/share/texlive/texmf-dist/tex/latex/hyperref/puenc.def +File: puenc.def 2023-02-07 v7.00v Hyperref: PDF Unicode definition (HO) +) +Package hyperref Info: Option `unicode' set `true' on input line 4060. +Package hyperref Info: Hyper figures OFF on input line 4177. +Package hyperref Info: Link nesting OFF on input line 4182. +Package hyperref Info: Hyper index ON on input line 4185. +Package hyperref Info: Plain pages OFF on input line 4192. +Package hyperref Info: Backreferencing OFF on input line 4197. +Package hyperref Info: Implicit mode ON; LaTeX internals redefined. +Package hyperref Info: Bookmarks ON on input line 4425. +\c@Hy@tempcnt=\count306 +(/usr/share/texlive/texmf-dist/tex/latex/url/url.sty +\Urlmuskip=\muskip18 +Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc. +) +LaTeX Info: Redefining \url on input line 4763. +\XeTeXLinkMargin=\dimen176 +(/usr/share/texlive/texmf-dist/tex/generic/bitset/bitset.sty +Package: bitset 2019/12/09 v1.3 Handle bit-vector datatype (HO) +(/usr/share/texlive/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty +Package: bigintcalc 2019/12/15 v1.5 Expandable calculations on big integers (HO +) +)) +\Fld@menulength=\count307 +\Field@Width=\dimen177 +\Fld@charsize=\dimen178 +Package hyperref Info: Hyper figures OFF on input line 6042. +Package hyperref Info: Link nesting OFF on input line 6047. +Package hyperref Info: Hyper index ON on input line 6050. +Package hyperref Info: backreferencing OFF on input line 6057. +Package hyperref Info: Link coloring OFF on input line 6062. +Package hyperref Info: Link coloring with OCG OFF on input line 6067. +Package hyperref Info: PDF/A mode OFF on input line 6072. +(/usr/share/texlive/texmf-dist/tex/latex/base/atbegshi-ltx.sty +Package: atbegshi-ltx 2021/01/10 v1.0c Emulation of the original atbegshi +package with kernel methods +) +\Hy@abspage=\count308 +\c@Item=\count309 +\c@Hfootnote=\count310 +) +Package hyperref Info: Driver (autodetected): hxetex. +(/usr/share/texlive/texmf-dist/tex/latex/hyperref/hxetex.def +File: hxetex.def 2023-02-07 v7.00v Hyperref driver for XeTeX +(/usr/share/texlive/texmf-dist/tex/generic/stringenc/stringenc.sty +Package: stringenc 2019/11/29 v1.12 Convert strings between diff. encodings (HO +) +) +\pdfm@box=\box63 +\c@Hy@AnnotLevel=\count311 +\HyField@AnnotCount=\count312 +\Fld@listcount=\count313 +\c@bookmark@seq@number=\count314 +(/usr/share/texlive/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty +Package: rerunfilecheck 2022-07-10 v1.10 Rerun checks for auxiliary files (HO) +(/usr/share/texlive/texmf-dist/tex/latex/base/atveryend-ltx.sty +Package: atveryend-ltx 2020/08/19 v1.0a Emulation of the original atveryend pac +kage +with kernel methods +) (/usr/share/texlive/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty +Package: uniquecounter 2019/12/15 v1.4 Provide unlimited unique counter (HO) +) +Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 2 +85. +) +\Hy@SectionHShift=\skip64 +) (/usr/share/texlive/texmf-dist/tex/latex/bookmark/bkm-dvipdfm.def +File: bkm-dvipdfm.def 2020-11-06 v1.29 bookmark driver for dvipdfm (HO) +\BKM@id=\count315 +)) +Package hyperref Info: Option `colorlinks' set `true' on input line 84. +(./BreedbaseManual.aux) +\openout1 = `BreedbaseManual.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for TU/lmr/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Checking defaults for PU/pdf/m/n on input line 90. +LaTeX Font Info: ... okay on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathrm' in version `normal' +(Font) OT1/lmr/m/n --> TU/lmr/m/n on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathit' in version `normal' +(Font) OT1/lmr/m/it --> TU/lmr/m/it on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathbf' in version `normal' +(Font) OT1/lmr/bx/n --> TU/lmr/bx/n on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `normal' +(Font) OT1/lmss/m/n --> TU/lmss/m/n on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `bold' +(Font) OT1/lmss/bx/n --> TU/lmss/bx/n on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `normal' +(Font) OT1/lmtt/m/n --> TU/lmtt/m/n on input line 90. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `bold' +(Font) OT1/lmtt/m/n --> TU/lmtt/bx/n on input line 90. + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: Font family 'latinmodern-math.otf(0)' created for font +(fontspec) 'latinmodern-math.otf' with options +(fontspec) [Scale=MatchLowercase,BoldItalicFont={},ItalicFont={},Sm +allCapsFont={},Script=Math,BoldFont={latinmodern-math.otf}]. +(fontspec) +(fontspec) This font family consists of the following NFSS +(fontspec) series/shapes: +(fontspec) +(fontspec) - 'normal' (m/n) with NFSS spec.: +(fontspec) <->s*[0.9999970497384594]"[latinmodern-math.otf]/OT:scri +pt=math;language=dflt;" +(fontspec) - 'small caps' (m/sc) with NFSS spec.: +(fontspec) - 'bold' (b/n) with NFSS spec.: +(fontspec) <->s*[0.9999970497384594]"[latinmodern-math.otf]/OT:scri +pt=math;language=dflt;" +(fontspec) - 'bold small caps' (b/sc) with NFSS spec.: + +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(0)/m/n' will be +(Font) scaled to size 12.0pt on input line 90. + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: Font family 'latinmodern-math.otf(1)' created for font +(fontspec) 'latinmodern-math.otf' with options +(fontspec) [Scale=MatchLowercase,BoldItalicFont={},ItalicFont={},Sm +allCapsFont={},Script=Math,SizeFeatures={{Size=10.2-},{Size=7.2-10.2,Font=latin +modern-math.otf,Style=MathScript},{Size=-7.2,Font=latinmodern-math.otf,Style=Ma +thScriptScript}},BoldFont={latinmodern-math.otf}]. +(fontspec) +(fontspec) This font family consists of the following NFSS +(fontspec) series/shapes: +(fontspec) +(fontspec) - 'normal' (m/n) with NFSS spec.: +(fontspec) <10.2->s*[0.9999970497384594]"[latinmodern-math.otf]/OT: +script=math;language=dflt;"<7.2-10.2>s*[0.9999970497384594]"[latinmodern-math.o +tf]/OT:script=math;language=dflt;+ssty=0;"<-7.2>s*[0.9999970497384594]"[latinmo +dern-math.otf]/OT:script=math;language=dflt;+ssty=1;" +(fontspec) - 'small caps' (m/sc) with NFSS spec.: +(fontspec) - 'bold' (b/n) with NFSS spec.: +(fontspec) <->s*[0.9999970497384594]"[latinmodern-math.otf]/OT:scri +pt=math;language=dflt;" +(fontspec) - 'bold small caps' (b/sc) with NFSS spec.: + +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 12.0pt on input line 90. +LaTeX Font Info: Encoding `OT1' has changed to `TU' for symbol font +(Font) `operators' in the math version `normal' on input line 90. +LaTeX Font Info: Overwriting symbol font `operators' in version `normal' +(Font) OT1/lmr/m/n --> TU/latinmodern-math.otf(1)/m/n on input + line 90. +LaTeX Font Info: Encoding `OT1' has changed to `TU' for symbol font +(Font) `operators' in the math version `bold' on input line 90. +LaTeX Font Info: Overwriting symbol font `operators' in version `bold' +(Font) OT1/lmr/bx/n --> TU/latinmodern-math.otf(1)/b/n on inpu +t line 90. + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 1.000097049443433. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 1.000097049443433. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 1.000097049443433. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 1.000097049443433. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 1.000097049443433. + + +Package fontspec Info: Font family 'latinmodern-math.otf(2)' created for font +(fontspec) 'latinmodern-math.otf' with options +(fontspec) [Scale=MatchLowercase,BoldItalicFont={},ItalicFont={},Sm +allCapsFont={},Script=Math,SizeFeatures={{Size=10.2-},{Size=7.2-10.2,Font=latin +modern-math.otf,Style=MathScript},{Size=-7.2,Font=latinmodern-math.otf,Style=Ma +thScriptScript}},BoldFont={latinmodern-math.otf},ScaleAgain=1.0001,FontAdjustme +nt={\fontdimen +(fontspec) 8\font =8.124pt\relax \fontdimen 9\font =4.728pt\relax +(fontspec) \fontdimen 10\font =5.328pt\relax \fontdimen 11\font +(fontspec) =8.232pt\relax \fontdimen 12\font =4.14pt\relax +(fontspec) \fontdimen 13\font =4.356pt\relax \fontdimen 14\font +(fontspec) =4.356pt\relax \fontdimen 15\font =3.468pt\relax +(fontspec) \fontdimen 16\font =2.964pt\relax \fontdimen 17\font +(fontspec) =2.964pt\relax \fontdimen 18\font =3.0pt\relax +(fontspec) \fontdimen 19\font =2.4pt\relax \fontdimen 22\font +(fontspec) =3.0pt\relax \fontdimen 20\font =0pt\relax \fontdimen +(fontspec) 21\font =0pt\relax }]. +(fontspec) +(fontspec) This font family consists of the following NFSS +(fontspec) series/shapes: +(fontspec) +(fontspec) - 'normal' (m/n) with NFSS spec.: +(fontspec) <10.2->s*[1.000097049443433]"[latinmodern-math.otf]/OT:s +cript=math;language=dflt;"<7.2-10.2>s*[1.000097049443433]"[latinmodern-math.otf +]/OT:script=math;language=dflt;+ssty=0;"<-7.2>s*[1.000097049443433]"[latinmoder +n-math.otf]/OT:script=math;language=dflt;+ssty=1;" +(fontspec) - 'small caps' (m/sc) with NFSS spec.: +(fontspec) and font adjustment code: +(fontspec) \fontdimen 8\font =8.124pt\relax \fontdimen 9\font +(fontspec) =4.728pt\relax \fontdimen 10\font =5.328pt\relax +(fontspec) \fontdimen 11\font =8.232pt\relax \fontdimen 12\font +(fontspec) =4.14pt\relax \fontdimen 13\font =4.356pt\relax +(fontspec) \fontdimen 14\font =4.356pt\relax \fontdimen 15\font +(fontspec) =3.468pt\relax \fontdimen 16\font =2.964pt\relax +(fontspec) \fontdimen 17\font =2.964pt\relax \fontdimen 18\font +(fontspec) =3.0pt\relax \fontdimen 19\font =2.4pt\relax \fontdimen +(fontspec) 22\font =3.0pt\relax \fontdimen 20\font =0pt\relax +(fontspec) \fontdimen 21\font =0pt\relax +(fontspec) - 'bold' (b/n) with NFSS spec.: +(fontspec) <->s*[1.000097049443433]"[latinmodern-math.otf]/OT:scrip +t=math;language=dflt;" +(fontspec) - 'bold small caps' (b/sc) with NFSS spec.: +(fontspec) and font adjustment code: +(fontspec) \fontdimen 8\font =8.124pt\relax \fontdimen 9\font +(fontspec) =4.728pt\relax \fontdimen 10\font =5.328pt\relax +(fontspec) \fontdimen 11\font =8.232pt\relax \fontdimen 12\font +(fontspec) =4.14pt\relax \fontdimen 13\font =4.356pt\relax +(fontspec) \fontdimen 14\font =4.356pt\relax \fontdimen 15\font +(fontspec) =3.468pt\relax \fontdimen 16\font =2.964pt\relax +(fontspec) \fontdimen 17\font =2.964pt\relax \fontdimen 18\font +(fontspec) =3.0pt\relax \fontdimen 19\font =2.4pt\relax \fontdimen +(fontspec) 22\font =3.0pt\relax \fontdimen 20\font =0pt\relax +(fontspec) \fontdimen 21\font =0pt\relax + +LaTeX Font Info: Encoding `OMS' has changed to `TU' for symbol font +(Font) `symbols' in the math version `normal' on input line 90. +LaTeX Font Info: Overwriting symbol font `symbols' in version `normal' +(Font) OMS/lmsy/m/n --> TU/latinmodern-math.otf(2)/m/n on inpu +t line 90. +LaTeX Font Info: Encoding `OMS' has changed to `TU' for symbol font +(Font) `symbols' in the math version `bold' on input line 90. +LaTeX Font Info: Overwriting symbol font `symbols' in version `bold' +(Font) OMS/lmsy/b/n --> TU/latinmodern-math.otf(2)/b/n on inpu +t line 90. + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9998970500334856. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9998970500334856. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9998970500334856. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9998970500334856. + + +Package fontspec Info: latinmodern-math scale = 0.9999970497384594. + + +Package fontspec Info: latinmodern-math scale = 0.9998970500334856. + + +Package fontspec Info: Font family 'latinmodern-math.otf(3)' created for font +(fontspec) 'latinmodern-math.otf' with options +(fontspec) [Scale=MatchLowercase,BoldItalicFont={},ItalicFont={},Sm +allCapsFont={},Script=Math,SizeFeatures={{Size=10.2-},{Size=7.2-10.2,Font=latin +modern-math.otf,Style=MathScript},{Size=-7.2,Font=latinmodern-math.otf,Style=Ma +thScriptScript}},BoldFont={latinmodern-math.otf},ScaleAgain=0.9999,FontAdjustme +nt={\fontdimen +(fontspec) 8\font =0.48pt\relax \fontdimen 9\font =2.4pt\relax +(fontspec) \fontdimen 10\font =2.004pt\relax \fontdimen 11\font +(fontspec) =1.332pt\relax \fontdimen 12\font =7.2pt\relax +(fontspec) \fontdimen 13\font =0pt\relax }]. +(fontspec) +(fontspec) This font family consists of the following NFSS +(fontspec) series/shapes: +(fontspec) +(fontspec) - 'normal' (m/n) with NFSS spec.: +(fontspec) <10.2->s*[0.9998970500334856]"[latinmodern-math.otf]/OT: +script=math;language=dflt;"<7.2-10.2>s*[0.9998970500334856]"[latinmodern-math.o +tf]/OT:script=math;language=dflt;+ssty=0;"<-7.2>s*[0.9998970500334856]"[latinmo +dern-math.otf]/OT:script=math;language=dflt;+ssty=1;" +(fontspec) - 'small caps' (m/sc) with NFSS spec.: +(fontspec) and font adjustment code: +(fontspec) \fontdimen 8\font =0.48pt\relax \fontdimen 9\font +(fontspec) =2.4pt\relax \fontdimen 10\font =2.004pt\relax +(fontspec) \fontdimen 11\font =1.332pt\relax \fontdimen 12\font +(fontspec) =7.2pt\relax \fontdimen 13\font =0pt\relax +(fontspec) - 'bold' (b/n) with NFSS spec.: +(fontspec) <->s*[0.9998970500334856]"[latinmodern-math.otf]/OT:scri +pt=math;language=dflt;" +(fontspec) - 'bold small caps' (b/sc) with NFSS spec.: +(fontspec) and font adjustment code: +(fontspec) \fontdimen 8\font =0.48pt\relax \fontdimen 9\font +(fontspec) =2.4pt\relax \fontdimen 10\font =2.004pt\relax +(fontspec) \fontdimen 11\font =1.332pt\relax \fontdimen 12\font +(fontspec) =7.2pt\relax \fontdimen 13\font =0pt\relax + +LaTeX Font Info: Encoding `OMX' has changed to `TU' for symbol font +(Font) `largesymbols' in the math version `normal' on input line 9 +0. +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `normal' +(Font) OMX/lmex/m/n --> TU/latinmodern-math.otf(3)/m/n on inpu +t line 90. +LaTeX Font Info: Encoding `OMX' has changed to `TU' for symbol font +(Font) `largesymbols' in the math version `bold' on input line 90. + +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `bold' +(Font) OMX/lmex/m/n --> TU/latinmodern-math.otf(3)/b/n on inpu +t line 90. +LaTeX Info: Redefining \microtypecontext on input line 90. +Package microtype Info: Applying patch `item' on input line 90. +Package microtype Info: Applying patch `toc' on input line 90. +Package microtype Info: Applying patch `eqnum' on input line 90. +Package microtype Info: Applying patch `footnote' on input line 90. +Package microtype Info: Applying patch `verbatim' on input line 90. +Package microtype Info: Character protrusion enabled (level 2). +Package microtype Info: Using protrusion set `basicmath'. +Package microtype Info: No adjustment of tracking. +Package microtype Info: No adjustment of spacing. +Package microtype Info: No adjustment of kerning. +(/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-LatinModernRoman.cfg +File: mt-LatinModernRoman.cfg 2021/02/21 v1.1 microtype config. file: Latin Mod +ern Roman (RS) +) +Package hyperref Info: Link coloring ON on input line 90. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 14.4pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 10.0pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 7.0pt on input line 92. +LaTeX Font Info: Trying to load font information for OML+lmm on input line 9 +2. +(/usr/share/texlive/texmf-dist/tex/latex/lm/omllmm.fd +File: omllmm.fd 2015/05/01 v1.6.1 Font defs for Latin Modern +) +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 14.4013pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 10.00092pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 7.00064pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 14.39845pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 9.99893pt on input line 92. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 6.99925pt on input line 92. +LaTeX Font Info: Trying to load font information for U+msa on input line 92. + +(/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsa.fd +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msa.cfg +File: mt-msa.cfg 2006/02/04 v1.1 microtype config. file: AMS symbols (a) (RS) +) +LaTeX Font Info: Trying to load font information for U+msb on input line 92. + +(/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsb.fd +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msb.cfg +File: mt-msb.cfg 2005/06/01 v1.0 microtype config. file: AMS symbols (b) (RS) +) [1 + + +] [2 + +] (./BreedbaseManual.toc +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 8.4pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(1)/m/n' will be +(Font) scaled to size 6.0pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 12.0011pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 8.40076pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(2)/m/n' will be +(Font) scaled to size 6.00055pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 11.99872pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 8.3991pt on input line 2. +LaTeX Font Info: Font shape `TU/latinmodern-math.otf(3)/m/n' will be +(Font) scaled to size 5.99936pt on input line 2. +[3] [4] [5] [6] [7] [8]) +\tf@toc=\write3 +\openout3 = `BreedbaseManual.toc'. + +[9] [10 + +] [11] [12 + +] +Chapter 1. +[13] +File: assets/images/image331.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[14] +File: assets/images/image166.png Graphic file (type bmp) + +[15] +File: assets/images/image261.png Graphic file (type bmp) + +File: assets/images/image144.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[16] +Underfull \vbox (badness 5417) has occurred while \output is active [] + +[17] +File: assets/images/image270.png Graphic file (type bmp) + +[18] [19] +File: assets/images/list_manager_start.png Graphic file (type bmp) + +File: assets/images/list_manager_new_list.png Graphic file (type bmp) + +[20] +File: assets/images/list_manager_view_list.png Graphic file (type bmp) + +File: assets/images/list_manager_add_items.png Graphic file (type bmp) + +[21] +File: assets/images/list_manager_added_items.png Graphic file (type bmp) + +File: assets/images/list_manager_list_types.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[22] +File: assets/images/image346.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[23] +File: assets/images/image71.png Graphic file (type bmp) + +File: assets/images/image297.png Graphic file (type bmp) + +[24] +File: assets/images/roles.png Graphic file (type bmp) + +[25] [26 + +] +Chapter 2. +File: assets/images/image267.png Graphic file (type bmp) + +File: assets/images/wizard_interface.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[27] +File: assets/images/wizard_interface_selections.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[28] +File: assets/images/wizard_select_list.png Graphic file (type bmp) + +File: assets/images/wizard_any_min_all_toggle.jpg Graphic file (type bmp) + +[29] +File: assets/images/wizard_any_min_all_toggle_min_details.jpg Graphic file (typ +e bmp) + +File: assets/images/add_create_list.png Graphic file (type bmp) + +[30] +File: assets/images/wizard_download_options.png Graphic file (type bmp) + +File: assets/images/wizard_related_metadata_download.png Graphic file (type bmp +) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[31] +File: assets/images/wizard_related_phenotypes_download.png Graphic file (type b +mp) + + +Underfull \vbox (badness 1748) has occurred while \output is active [] + +[32] +File: assets/images/wizard_load_create_dataset.png Graphic file (type bmp) + +File: assets/images/wizard_update_list_refresh.png Graphic file (type bmp) + + +Underfull \vbox (badness 7685) has occurred while \output is active [] + +[33] +File: assets/images/search_accessions.png Graphic file (type bmp) + +File: assets/images/search_accessions_properties_search.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[34] +File: assets/images/search_accessions_properties_view.png Graphic file (type bm +p) + +File: assets/images/search_accessions_graphical_filtering.png Graphic file (typ +e bmp) + + +Underfull \vbox (badness 8132) has occurred while \output is active [] + +[35] +File: assets/images/trial_search.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[36] +File: assets/images/trait-search-default.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[37] +File: assets/images/trait-search.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[38] +File: assets/images/ontology_browser.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[39] +File: assets/images/ontology_browser_variable.png Graphic file (type bmp) + +File: assets/images/search_seedlots.png Graphic file (type bmp) + + +Underfull \vbox (badness 7595) has occurred while \output is active [] + +[40] +File: assets/images/manage_seedlots.png Graphic file (type bmp) + +[41] [42 + +] +Chapter 3. +File: assets/images/manage_user_roles_page.png Graphic file (type bmp) + +[43] [44] +Chapter 4. +File: assets/images/breeding_programs_screenshot.png Graphic file (type bmp) + +File: assets/images/new_program_screenshot.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[45 + +] [46] +Chapter 5. +File: assets/images/image141.png Graphic file (type bmp) + +[47 + +] +File: assets/images/image187.png Graphic file (type bmp) + +File: assets/images/location_map.png Graphic file (type bmp) + +[48] +Chapter 6. +File: assets/images/manage_accessions_add_accessions_link.png Graphic file (typ +e bmp) + +File: assets/images/manage_accessions_add_accessions_using_lists.png Graphic fi +le (type bmp) + +[49 + +] +File: assets/images/manage_accessions_add_accessions_found.png Graphic file (ty +pe bmp) + +File: assets/images/manage_accessions_add_accessions_fuzzy.png Graphic file (ty +pe bmp) + +[50] +File: assets/images/manage_accessions_add_accessions_fuzzy_options.png Graphic +file (type bmp) + +File: assets/images/manage_accessions_add_accessions_complete_using_list.png Gr +aphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[51] +File: assets/images/manage_accessions_add_accessions_saved.png Graphic file (ty +pe bmp) + + +Underfull \hbox (badness 1990) in paragraph at lines 671--671 +[]\TU/lmr/bx/n/17.28 Uploading Accessions and Accession’s + [] + +File: assets/images/manage_accessions_add_accessions_using_file.png Graphic fil +e (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[52] +File: assets/images/manage_accessions_add_accessions_spreadsheet_info.png Graph +ic file (type bmp) + + +Underfull \vbox (badness 1838) has occurred while \output is active [] + + +Overfull \hbox (20.4643pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 6.2. UPLOADING ACCESSIONS AND ACCESSION’S INFO FROM A FILE \TU +/lmr/m/n/12 53 + [] + +[53] +File: assets/images/manage_accessions_add_accessions_complete_using_file.png Gr +aphic file (type bmp) + +File: assets/images/manage_accessions_add_accessions_saved.png Graphic file (ty +pe bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[54] +File: assets/images/accession_upload_using_email.png Graphic file (type bmp) + + +Underfull \vbox (badness 2582) has occurred while \output is active [] + +[55] +File: assets/images/image286.png Graphic file (type bmp) + +File: assets/images/image333.png Graphic file (type bmp) + +File: assets/images/pedigree_upload_format.png Graphic file (type bmp) + + +Underfull \vbox (badness 8038) has occurred while \output is active [] + +[56] [57] [58] [59] [60 + +] +Chapter 7. +File: assets/images/manage_seedlots.png Graphic file (type bmp) + + +Underfull \vbox (badness 4378) has occurred while \output is active [] + +[61] +File: assets/images/manage_seedlot_add_seedlot.png Graphic file (type bmp) + +File: assets/images/manage_seedlot_upload_seedlots.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[62] +File: assets/images/seedlot_detail_page.png Graphic file (type bmp) + +[63] +File: assets/images/seedlot_add_new_transaction.png Graphic file (type bmp) + +File: assets/images/manage_seedlot_upload_inventory.png Graphic file (type bmp) + + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[64] +File: assets/images/manage_seedlot_upload_inventory_template.png Graphic file ( +type bmp) + +[65] +File: assets/images/manage_seedlot_accession_list_search.png Graphic file (type + bmp) + +File: assets/images/manage_seedlot_list_available_seedlots.png Graphic file (ty +pe bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[66] +File: assets/images/manage_seedlot_accession_detail.png Graphic file (type bmp) + + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[67] +File: assets/images/manage_seedlots_seedlot_maintenance.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[68] [69] +File: assets/images/seedlot_maintenance_upload.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[70] [71] +Underfull \hbox (badness 1337) in paragraph at lines 914--916 +[]\TU/lmr/m/n/12 Optionally, notes button next to each event to add additional + [] + +File: assets/images/seedlot_maintenance_record.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[72] +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[73] +File: assets/images/seedlot_maintenance_events_unfiltered.png Graphic file (typ +e bmp) + +[74] +File: assets/images/seedlot_maintenance_events_filtered.png Graphic file (type +bmp) + +[75] [76] +Chapter 8. +File: assets/images/manage_populations.png Graphic file (type bmp) + +File: assets/images/manage_populations_add_population.png Graphic file (type bm +p) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[77 + +] +File: assets/images/manage_populations_expand_table.png Graphic file (type bmp) + + +File: assets/images/manage_populations_table.png Graphic file (type bmp) + +[78] +Chapter 9. +File: assets/images/image148.png Graphic file (type bmp) + +File: assets/images/manage_crosses1.png Graphic file (type bmp) + + +Underfull \vbox (badness 2277) has occurred while \output is active [] + +[79 + +] +File: assets/images/manage_crosses2.png Graphic file (type bmp) + +File: assets/images/manage_crosses_crossingtrial.png Graphic file (type bmp) + +[80] +File: assets/images/add_crossingtrial.png Graphic file (type bmp) + +File: assets/images/manage_crosses_addcross.png Graphic file (type bmp) + +[81] +File: assets/images/add_cross.png Graphic file (type bmp) + +[82] +File: assets/images/new_cross_dialog_type.png Graphic file (type bmp) + +File: assets/images/new_cross_dialog2.png Graphic file (type bmp) + +[83] +File: assets/images/add_cross_info.png Graphic file (type bmp) + +File: assets/images/manage_crosses_upload.png Graphic file (type bmp) + +File: assets/images/upload_crosses.png Graphic file (type bmp) + + +Underfull \vbox (badness 5711) has occurred while \output is active [] + +[84] +File: assets/images/cross_upload_format.png Graphic file (type bmp) + +[85] +File: assets/images/cross_upload_page.png Graphic file (type bmp) + +File: assets/images/progenies_upload_spreadsheet.png Graphic file (type bmp) + +File: assets/images/crossinfo_upload_spreadsheet.png Graphic file (type bmp) + +[86] +File: assets/images/manage_cross_click_cross_wishlist.png Graphic file (type bm +p) + +File: assets/images/cross_wishlist_initial_dialog.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[87] +File: assets/images/cross_wishlist_not_using_list_01.png Graphic file (type bmp +) + +File: assets/images/cross_wishlist_not_using_list_02.png Graphic file (type bmp +) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[88] +File: assets/images/cross_wishlist_not_using_list_03.png Graphic file (type bmp +) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[89] +File: assets/images/cross_wishlist_using_list_01.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[90] +File: assets/images/cross_wishlist_using_list_02.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[91] +File: assets/images/cross_wishlist_not_using_list_04.png Graphic file (type bmp +) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[92] +File: assets/images/crossingtrial_1.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[93] +File: assets/images/crossingtrial_2.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[94] +File: assets/images/cross_page_1.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[95] +File: assets/images/cross_page_2.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[96] +File: assets/images/edit_cross_info.png Graphic file (type bmp) + +File: assets/images/cross_add_progenies.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[97] [98] +Chapter 10. +File: assets/images/image290.png Graphic file (type bmp) + +File: assets/images/image153.png Graphic file (type bmp) + +File: assets/images/image279.png Graphic file (type bmp) + +[99 + +] +File: assets/images/trial_detail_page_start.png Graphic file (type bmp) + +[100] +File: assets/images/trial_detail_page_navigator_1.png Graphic file (type bmp) + +File: assets/images/trial_detail_page_navigator_2.png Graphic file (type bmp) + +File: assets/images/trial_detail_page_navigator_3.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[101] +File: assets/images/add_transplanting_date.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[102] [103] +File: assets/images/trial_create_manage_trials.png Graphic file (type bmp) + +File: assets/images/trial_create_open_form.png Graphic file (type bmp) + +File: assets/images/trial_create_form_1.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[104] +File: assets/images/trial_create_form_2.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[105] +File: assets/images/trial_create_form_3.png Graphic file (type bmp) + +[106] +File: assets/images/trial_create_form_4.png Graphic file (type bmp) + +[107] +File: assets/images/trial_create_form_5.png Graphic file (type bmp) + +File: assets/images/trial_create_form_6.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[108] +File: assets/images/trial_create_form_7_top.png Graphic file (type bmp) + +[109] +File: assets/images/trial_create_form_7_middle.png Graphic file (type bmp) + +File: assets/images/trial_create_form_7_bottom.png Graphic file (type bmp) + + +Underfull \vbox (badness 1590) has occurred while \output is active [] + +[110] +File: assets/images/add_management_factor_name_dialog.png Graphic file (type bm +p) + +File: assets/images/add_management_factor_dialog.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[111] +File: assets/images/trial_create_form_8.png Graphic file (type bmp) + +[112] +File: assets/images/trial_create_manage_trials.png Graphic file (type bmp) + +File: assets/images/manage_trials_upload_trial_1.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[113] +File: assets/images/manage_trials_upload_trial_2.png Graphic file (type bmp) + +File: assets/images/manage_trials_upload_trial_template.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[114] +Overfull \hbox (33.10004pt too wide) in paragraph at lines 1371--1373 +\TU/lmr/m/n/12 range_number row_number col_number seedlot_name num_seed_per_plo +t + [] + + +Overfull \hbox (49.91434pt too wide) in paragraph at lines 1394--1395 +[]\TU/lmr/bx/n/12 plot_name + [] + + +Overfull \hbox (75.33476pt too wide) in paragraph at lines 1396--1397 +[]\TU/lmr/bx/n/12 accession_name + [] + + +Overfull \hbox (61.62901pt too wide) in paragraph at lines 1398--1399 +[]\TU/lmr/bx/n/12 plot_number + [] + + +Overfull \hbox (68.56636pt too wide) in paragraph at lines 1400--1401 +[]\TU/lmr/bx/n/12 block_number + [] + + +Overfull \hbox (58.45169pt too wide) in paragraph at lines 1402--1403 +[]\TU/lmr/bx/n/12 is_a_control + [] + + +Overfull \hbox (58.47168pt too wide) in paragraph at lines 1404--1405 +[]\TU/lmr/bx/n/12 rep_number + [] + + +Overfull \hbox (69.92235pt too wide) in paragraph at lines 1406--1407 +[]\TU/lmr/bx/n/12 range_number + [] + + +Overfull \hbox (60.95567pt too wide) in paragraph at lines 1408--1409 +[]\TU/lmr/bx/n/12 row_number + [] + + +Overfull \hbox (55.80768pt too wide) in paragraph at lines 1410--1411 +[]\TU/lmr/bx/n/12 col_number + [] + + +Overfull \hbox (64.75035pt too wide) in paragraph at lines 1412--1413 +[]\TU/lmr/bx/n/12 seedlot_name + [] + + +Overfull \hbox (100.03745pt too wide) in paragraph at lines 1414--1415 +[]\TU/lmr/bx/n/12 num_seed_per_plot + [] + + +Overfull \hbox (144.42093pt too wide) in paragraph at lines 1416--1417 +[]\TU/lmr/bx/n/12 weight_gram_seed_per_plot + [] + + +Overfull \hbox (69.54901pt too wide) in paragraph at lines 1418--1419 +[]\TU/lmr/bx/n/12 entry_number + [] + + +Overfull \hbox (34.61433pt too wide) in paragraph at lines 1424--1424 +[]|\TU/lmr/m/n/12 2018plot1| + [] + + +Overfull \hbox (57.10675pt too wide) in paragraph at lines 1424--1424 +[]|\TU/lmr/m/n/12 my_accession1| + [] + + +Overfull \hbox (0.74101pt too wide) in paragraph at lines 1424--1424 +[]|\TU/lmr/m/n/12 101| + [] + + +Overfull \hbox (34.61433pt too wide) in paragraph at lines 1425--1425 +[]|\TU/lmr/m/n/12 2018plot2| + [] + + +Overfull \hbox (57.10675pt too wide) in paragraph at lines 1425--1425 +[]|\TU/lmr/m/n/12 my_accession2| + [] + + +Overfull \hbox (0.74101pt too wide) in paragraph at lines 1425--1425 +[]|\TU/lmr/m/n/12 201| + [] + + +Overfull \hbox (34.61433pt too wide) in paragraph at lines 1426--1426 +[]|\TU/lmr/m/n/12 2018plot3| + [] + + +Overfull \hbox (57.10675pt too wide) in paragraph at lines 1426--1426 +[]|\TU/lmr/m/n/12 my_accession2| + [] + + +Overfull \hbox (0.74101pt too wide) in paragraph at lines 1426--1426 +[]|\TU/lmr/m/n/12 102| + [] + + +Overfull \hbox (34.61433pt too wide) in paragraph at lines 1427--1427 +[]|\TU/lmr/m/n/12 2018plot4| + [] + + +Overfull \hbox (57.10675pt too wide) in paragraph at lines 1427--1427 +[]|\TU/lmr/m/n/12 my_accession1| + [] + + +Overfull \hbox (0.74101pt too wide) in paragraph at lines 1427--1427 +[]|\TU/lmr/m/n/12 202| + [] + +[115] +Underfull \hbox (badness 1072) in paragraph at lines 1430--1431 +\TU/lmr/m/n/12 ing_program location year transplanting_date design_type descrip +tion + [] + + +Underfull \hbox (badness 3758) in paragraph at lines 1430--1431 +\TU/lmr/m/n/12 trial_type trial_stock_type plot_width plot_length field_size pl +ant- + [] + +File: assets/images/manage_trials_upload_trial_3.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[116] +File: assets/images/manage_trials_upload_trial_4.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[117] +File: assets/images/manage_trials_upload_trial_5.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[118] [119] +File: assets/images/manage_trials_upload_trial_7.png Graphic file (type bmp) + +File: assets/images/manage_trials_upload_trial_8.png Graphic file (type bmp) + +[120] +File: assets/images/multiple_trial_upload_with_email.png Graphic file (type bmp +) + +[121] +File: assets/images/fieldmap_trial_layout.png Graphic file (type bmp) + +File: assets/images/fieldmap_multi_trial_layout.png Graphic file (type bmp) + +[122] +File: assets/images/fieldmap_plot_image.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[123] +File: assets/images/fieldmap_view_plot_image.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[124] +File: assets/images/fieldmap_display_plot_image.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[125] +File: assets/images/heatmap_assayed_trait_view.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[126] +File: assets/images/trial_detail_page_suppress_phenotype.png Graphic file (type + bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[127] +File: assets/images/heatmap_spatial_correction_button.png Graphic file (type bm +p) + +[128] +File: assets/images/image332.png Graphic file (type bmp) + +File: assets/images/image340.png Graphic file (type bmp) + +File: assets/images/image190.png Graphic file (type bmp) + +[129] +File: assets/images/image79.png Graphic file (type bmp) + +File: assets/images/image276.png Graphic file (type bmp) + +File: assets/images/field_map_download_link.png Graphic file (type bmp) + +[130] +File: assets/images/spatial_layout_usage_help.png Graphic file (type bmp) + +File: assets/images/spatial_layout_usage_help_1.png Graphic file (type bmp) + +File: assets/images/spatial_layout_usage_help_2.png Graphic file (type bmp) + +File: assets/images/spatial_layout_usage_help_3.png Graphic file (type bmp) + +[131] +File: assets/images/fieldmap_view_plot_image.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[132] +File: assets/images/replace_plot_accession_form.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[133] +File: assets/images/image173.png Graphic file (type bmp) + +File: assets/images/image87.png Graphic file (type bmp) + +File: assets/images/image85.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[134] +File: assets/images/image348.png Graphic file (type bmp) + +File: assets/images/image347.png Graphic file (type bmp) + +[135] [136] +File: assets/images/trial_detail_page_add_plant_entries.png Graphic file (type +bmp) + +File: assets/images/trial_detail_page_add_plant_entries_autogenerated.png Graph +ic file (type bmp) + +[137] +File: assets/images/manage_trials_tissue_sample_default.png Graphic file (type +bmp) + +File: assets/images/manage_trials_tissue_sample_create.png Graphic file (type b +mp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[138] +File: assets/images/manage_trials_tissue_samples.png Graphic file (type bmp) + +File: assets/images/manage_trials_tissue_sample_detail.png Graphic file (type b +mp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[139] +File: assets/images/manage_trials_tissue_sample_related_stock.png Graphic file +(type bmp) + +File: assets/images/trial_detail_page_add_plot_gps.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[140] +File: assets/images/trial_detail_page_add_plot_gps_dialog.png Graphic file (typ +e bmp) + +File: assets/images/trial_detail_page_add_plot_gps_upload_info.png Graphic file + (type bmp) + + +Underfull \hbox (badness 4556) in paragraph at lines 1762--1763 +\TU/lmr/m/n/12 contain: plot_name WGS84_bottom_left_x WGS84_bottom_left_y + [] + + +Underfull \vbox (badness 1603) has occurred while \output is active [] + +[141] +File: assets/images/trial_detail_page_view_repetitive_measurements.png Graphic +file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[142] +File: assets/images/trial_upload_additional_file.png Graphic file (type bmp) + +File: assets/images/trial_upload_additional_file_info.png Graphic file (type bm +p) + +File: assets/images/update_trial_metadata.png Graphic file (type bmp) + +[143] [144] +File: assets/images/trial_detail_page_delete_trait1.png Graphic file (type bmp) + + +File: assets/images/trial_detail_page_delete_trait2.png Graphic file (type bmp) + + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[145] +File: assets/images/trial_detail_page_delete_trait3.png Graphic file (type bmp) + + +[146] +Chapter 11. +File: assets/images/manage_genotyping_trials.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[147 + +] +File: assets/images/manage_genotyping_trials_expand.png Graphic file (type bmp) + + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[148] +File: assets/images/manage_genotyping_trials_add_trial.png Graphic file (type b +mp) + +File: assets/images/manage_genotyping_trials_add_trial_wells.png Graphic file ( +type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[149] +File: assets/images/manage_genotyping_trials_upload_template.png Graphic file ( +type bmp) + +[150] +File: assets/images/manage_genotyping_trials_add_trial_confirm.png Graphic file + (type bmp) + +File: assets/images/manage_genotyping_trials_detail_info.png Graphic file (type + bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[151] +File: assets/images/manage_genotyping_trials_detail_table.png Graphic file (typ +e bmp) + +[152] +Chapter 12. +File: assets/images/FB_Section_Header.png Graphic file (type bmp) + +[153 + +] +File: assets/images/QR_FieldBook_PlayStore.png Graphic file (type bmp) + + +Overfull \hbox (167.5337pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 154 \TU/lmr/m/sl/12 CHAPTER 12. USING FIELD BOOK AND BREEDBASE +FOR DATA COLLECTION AND STORAGE + [] + +[154] [155] +Underfull \hbox (badness 1675) in paragraph at lines 1984--1984 +[]\TU/lmr/bx/n/17.28 Adding Traits to Field Book Using + [] + + +Underfull \hbox (badness 1803) in paragraph at lines 2003--2003 +[]\TU/lmr/bx/n/17.28 Collecting and Exporting Phenotype + [] + + +Overfull \hbox (167.5337pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 156 \TU/lmr/m/sl/12 CHAPTER 12. USING FIELD BOOK AND BREEDBASE +FOR DATA COLLECTION AND STORAGE + [] + +[156] +Overfull \hbox (115.54572pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 12.7. USING BRAPI FOR DATA TRANSFER BETWEEN FIELD BOOK AND BRE +EDBASE \TU/lmr/m/n/12 157 + [] + +[157] +Overfull \hbox (167.5337pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 158 \TU/lmr/m/sl/12 CHAPTER 12. USING FIELD BOOK AND BREEDBASE +FOR DATA COLLECTION AND STORAGE + [] + +[158] +Overfull \hbox (115.54572pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 12.7. USING BRAPI FOR DATA TRANSFER BETWEEN FIELD BOOK AND BRE +EDBASE \TU/lmr/m/n/12 159 + [] + +[159] +Overfull \hbox (167.5337pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 160 \TU/lmr/m/sl/12 CHAPTER 12. USING FIELD BOOK AND BREEDBASE +FOR DATA COLLECTION AND STORAGE + [] + +[160 + +] +Chapter 13. +File: assets/images/image178.png Graphic file (type bmp) + +[161] +File: assets/images/image185.png Graphic file (type bmp) + + +LaTeX Warning: Reference `using-fieldbook-app' on page 162 undefined on input l +ine 2171. + +File: assets/images/image112.png Graphic file (type bmp) + +File: assets/images/image288.png Graphic file (type bmp) + +File: assets/images/image68.png Graphic file (type bmp) + +[162] +File: assets/images/phenotype_spreadsheet_upload_help_dialog.png Graphic file ( +type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[163] +File: assets/images/create_phenotype_spreadsheet_dialog.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[164] +File: assets/images/phenotype_spreadsheet_upload_dialog.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[165] [166] +Chapter 14. +File: assets/images/transcriptomics_menu.png Graphic file (type bmp) + +[167 + +] +File: assets/images/create_transcriptomics_protocol.png Graphic file (type bmp) + + +File: assets/images/data_matrix_headers.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + + +Overfull \hbox (30.43217pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 168 \TU/lmr/m/sl/12 CHAPTER 14. MANAGING HIGH DIMENSIONAL PHENO +TYPING DATA + [] + +[168] +Underfull \hbox (badness 1286) in paragraph at lines 2225--2226 +\TU/lmr/m/n/12 you created in your sampling trial file. The header marked “tran +- + [] + +File: assets/images/data_matrix_example.png Graphic file (type bmp) + +File: assets/images/transcript_details_headers.png Graphic file (type bmp) + +File: assets/images/transcript_details_example.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[169] +File: assets/images/download_transcriptomics.png Graphic file (type bmp) + + +Overfull \hbox (30.43217pt too wide) has occurred while \output is active +\TU/lmr/m/n/12 170 \TU/lmr/m/sl/12 CHAPTER 14. MANAGING HIGH DIMENSIONAL PHENO +TYPING DATA + [] + +[170] +Chapter 15. +File: assets/images/image82.png Graphic file (type bmp) + +File: assets/images/image274.png Graphic file (type bmp) + + +Underfull \vbox (badness 2189) has occurred while \output is active [] + +[171 + +] +File: assets/images/image245.png Graphic file (type bmp) + +File: assets/images/image264.png Graphic file (type bmp) + +[172] +File: assets/images/image248.png Graphic file (type bmp) + +File: assets/images/plot_pheno_barcode.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[173] +File: assets/images/trial_barcode.png Graphic file (type bmp) + +[174] +Chapter 16. +File: assets/gifs/gif1.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[175 + +] +File: assets/gifs/gif2.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[176] +File: assets/gifs/gif3.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[177] +File: assets/gifs/gif4.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[178] [179] [180 + +] +Chapter 17. +File: assets/images/image255.png Graphic file (type bmp) + +File: assets/images/trial_meta_data_download.png Graphic file (type bmp) + +File: assets/images/image287.png Graphic file (type bmp) + +[181] +File: assets/images/image182.png Graphic file (type bmp) + +[182] +Chapter 18. +File: assets/images/odk_ona_management.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[183 + +] +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[184] +File: assets/images/odk_ona_cross_status.png Graphic file (type bmp) + +File: assets/images/odk_ona_cross_progress_graph.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[185] +File: assets/images/odk_ona_cross_progress_tree.png Graphic file (type bmp) + +[186] +Chapter 19. +File: assets/images/manage_tissue_samples_create_tissues_field_trial_download.p +ng Graphic file (type bmp) + +[187 + +] +File: assets/images/manage_tissue_samples_create_tissues_field_trial_button.png + Graphic file (type bmp) + +File: assets/images/manage_tissue_samples_create_tissues_field_trial_intro.png +Graphic file (type bmp) + +File: assets/images/manage_tissue_samples_create_tissues_field_trial_select.png + Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[188] +File: assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants +.png Graphic file (type bmp) + + +File: assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissue +s.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[189] +File: assets/images/manage_tissue_samples_create_tissues_field_trial_submit.png + Graphic file (type bmp) + +[190] +File: assets/images/manage_tissue_samples_create_tissues_genotyping_trial_downl +oad.png Graphic file (type bmp) + +File: assets/images/manage_tissue_samples_create_tissues_genotyping_trial_butto +n.png Graphic file (type bmp) + + +Overfull \hbox (38.18161pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 19.2. GENOTYPING PLATE TISSUE SAMPLES (96 OR 384 WELL PLATES) +\TU/lmr/m/n/12 191 + [] + +[191] [192 + +] +Chapter 20. +File: assets/images/manage_observation_variables_start_button.png Graphic file +(type bmp) + +[193] +File: assets/images/manage_observation_variables_workflow_intro.png Graphic fil +e (type bmp) + +File: assets/images/manage_observation_variables_workflow_observation_variable. +png Graphic file (type bmp) + +[194] +File: assets/images/manage_observation_variables_workflow_trait.png Graphic fil +e (type bmp) + +File: assets/images/manage_observation_variables_workflow_method.png Graphic fi +le (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + + +Overfull \hbox (117.5829pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 20.1. MANAGING OBSERVATION VARIABLES WITH TRAITS, METHODS, AND + SCALES \TU/lmr/m/n/12 195 + [] + +[195] +File: assets/images/manage_observation_variables_workflow_scale.png Graphic fil +e (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[196] +File: assets/images/manage_observation_variables_workflow_submit.png Graphic fi +le (type bmp) + + +Overfull \hbox (117.5829pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 20.1. MANAGING OBSERVATION VARIABLES WITH TRAITS, METHODS, AND + SCALES \TU/lmr/m/n/12 197 + [] + +[197] [198 + +] +Chapter 21. +File: assets/images/manage_image_phenotyping_upload_intro.png Graphic file (typ +e bmp) + + +Underfull \vbox (badness 7486) has occurred while \output is active [] + +[199] +File: assets/images/manage_image_phenotyping_upload_field_trial.png Graphic fil +e (type bmp) + +File: assets/images/manage_image_phenotyping_upload_drone_run.png Graphic file +(type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[200] +File: assets/images/manage_image_phenotyping_upload_image_info_1.png Graphic fi +le (type bmp) + + +Underfull \vbox (badness 1762) has occurred while \output is active [] + +[201] +File: assets/images/manage_image_phenotyping_upload_image_info_2.png Graphic fi +le (type bmp) + +File: assets/images/manage_image_phenotyping_upload_images_zipfile.png Graphic +file (type bmp) + +[202] +File: assets/images/manage_image_phenotyping_upload_image_info_3.png Graphic fi +le (type bmp) + +File: assets/images/manage_image_phenotyping_upload_images_rasters.png Graphic +file (type bmp) + +[203] +File: assets/images/manage_image_phenotyping_standard_process_button.png Graphi +c file (type bmp) + +File: assets/images/manage_image_phenotyping_standard_process_intro.png Graphic + file (type bmp) + +[204] +File: assets/images/manage_image_phenotyping_standard_process_drone_run_band.pn +g Graphic file (type bmp) + +File: assets/images/manage_image_phenotyping_standard_process_rotate.png Graphi +c file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[205] +File: assets/images/manage_image_phenotyping_standard_process_cropping.png Grap +hic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[206] +File: assets/images/manage_image_phenotyping_standard_process_threshold_hist.pn +g Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[207] +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[208] +File: assets/images/manage_image_phenotyping_standard_process_plot_polygon.png +Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[209] [210] +File: assets/images/manage_image_phenotyping_standard_process_apply.png Graphic + file (type bmp) + +File: assets/images/manage_image_phenotyping_standard_process_vegetation_index. +png Graphic file (type bmp) + +File: assets/images/manage_image_phenotyping_standard_process_phenotypes.png Gr +aphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[211] [212] +Chapter 22. +File: assets/images/image_upload_dialog.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[213 + +] +File: assets/images/filename_verify_success.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[214] +Underfull \vbox (badness 2401) has occurred while \output is active [] + +[215] +File: assets/images/stock_add_image.png Graphic file (type bmp) + +[216] +Chapter 23. +File: assets/images/manage_genotyping_data_upload_button.png Graphic file (type + bmp) + +[217 + +] +File: assets/images/manage_genotyping_data_upload_dialog_intro.png Graphic file + (type bmp) + +File: assets/images/manage_genotyping_data_upload_dialog_project.png Graphic fi +le (type bmp) + + +Underfull \vbox (badness 2884) has occurred while \output is active [] + +[218] +File: assets/images/manage_genotyping_data_upload_dialog_protocol.png Graphic f +ile (type bmp) + +File: assets/images/manage_genotyping_data_upload_dialog_vcf.png Graphic file ( +type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[219] +File: assets/images/manage_genotyping_data_wizard_download_prot.png Graphic fil +e (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[220] +File: assets/images/manage_genotyping_data_wizard_trial.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[221] +File: assets/images/search_wizard_genotyping_protocols_download_main.png Graphi +c file (type bmp) + +[222] +File: assets/images/search_wizard_genotyping_protocols_download_genotypes.png G +raphic file (type bmp) + +File: assets/images/search_wizard_genotyping_protocols_download_phenotypes.png +Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[223] +File: assets/images/genotyping_protocol_detail_page_markers.png Graphic file (t +ype bmp) + +[224] +File: assets/images/genotyping_protocol_detail_page_samples.png Graphic file (t +ype bmp) + +File: assets/images/genotyping_protocol_detail_page_delete.png Graphic file (ty +pe bmp) + +[225] [226 + +] +Chapter 24. +File: assets/images/waves_breedbase_schema.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[227] +File: assets/images/manage_NIRS_main_display.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[228] +File: assets/images/upload_NIRS_example_format.png Graphic file (type bmp) + +[229] +File: assets/images/upload_NIRS_plot_example.png Graphic file (type bmp) + +[230] [231] [232 + +] +Chapter 25. +File: assets/images/sequence_metadata_manage.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[233] [234] +File: assets/images/sequence_metadata_search_basic.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[235] +File: assets/images/sequence_metadata_search_results.png Graphic file (type bmp +) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[236] +File: assets/images/sequence_metadata_search_advanced.png Graphic file (type bm +p) + + +Underfull \hbox (badness 3838) in paragraph at lines 2877--2878 +\TU/lmr/m/n/12 A table of Sequence Metadata annotations are embedded on the + [] + +[237] [238 + +] +Chapter 26. + +Underfull \hbox (badness 7558) in paragraph at lines 2888--2888 +[]\TU/lmr/bx/n/17.28 What is Outliers Functionality in + [] + +File: assets/images/outliers_dataset_icon.png Graphic file (type bmp) + +[239] +File: assets/images/outliers_dataset_basic_panel.png Graphic file (type bmp) + +File: assets/images/outliers_dataset_full_visualisation.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[240] +Underfull \vbox (badness 3049) has occurred while \output is active [] + +[241] +File: assets/images/outliers_dataset_actions.png Graphic file (type bmp) + +[242] +Chapter 27. +File: assets/images/image114.png Graphic file (type bmp) + +File: assets/images/image251.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[243 + +] +File: assets/images/image95.png Graphic file (type bmp) + +File: assets/images/image78.png Graphic file (type bmp) + +File: assets/images/image304.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[244] +File: assets/images/image76.png Graphic file (type bmp) + +File: assets/images/image343.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[245] +File: assets/images/image326.png Graphic file (type bmp) + +File: assets/images/image150.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[246] +File: assets/images/image156.png Graphic file (type bmp) + +File: assets/images/image247.png Graphic file (type bmp) + +[247] +File: assets/images/image160.png Graphic file (type bmp) + +[248] +File: assets/images/image249.png Graphic file (type bmp) + +[249] +File: assets/images/image330.png Graphic file (type bmp) + +File: assets/images/image244.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[250] +File: assets/images/image263.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[251] +File: assets/images/image83.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[252] +File: assets/images/image93.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[253] +File: assets/images/image328.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[254] +File: assets/images/image74.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[255] +File: assets/images/image147.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[256] [257] +File: assets/images/image338.png Graphic file (type bmp) + +File: assets/images/image334.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[258] [259] +File: assets/images/image344.png Graphic file (type bmp) + +File: assets/images/image329.png Graphic file (type bmp) + +File: assets/images/image341.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[260] +File: assets/images/image322.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[261] +File: assets/images/image151.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[262] +File: assets/images/image69.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[263] +File: assets/images/image336.png Graphic file (type bmp) + + +Underfull \vbox (badness 1389) has occurred while \output is active [] + +[264] +File: assets/images/image171.png Graphic file (type bmp) + +[265] +File: assets/images/image81.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[266] +File: assets/images/image138.png Graphic file (type bmp) + +File: assets/images/image181.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[267] +File: assets/images/image152.png Graphic file (type bmp) + +[268] +File: assets/images/image20.png Graphic file (type bmp) + +File: assets/images/image318.png Graphic file (type bmp) + +File: assets/images/image277.png Graphic file (type bmp) + + +Underfull \vbox (badness 4156) has occurred while \output is active [] + +[269] +File: assets/images/image268.png Graphic file (type bmp) + +File: assets/images/image327.png Graphic file (type bmp) + +[270] +File: assets/images/pca_iita_naccri_trials.png Graphic file (type bmp) + +[271] +File: assets/images/anova-dm.png Graphic file (type bmp) + +[272] +File: assets/images/k-means-cluster.png Graphic file (type bmp) + + +Underfull \vbox (badness 6927) has occurred while \output is active [] + +[273] +File: assets/images/hclustering.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[274] +File: assets/images/genetic-gain.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[275] [276] +File: assets/images/kinship-inbreeding.png Graphic file (type bmp) + +[277] +File: assets/images/selection_proportion_clustering.png Graphic file (type bmp) + + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[278] +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[279] +File: assets/images/search_wizard_genotype_analyses_grm.png Graphic file (type +bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[280] +Overfull \hbox (34.74158pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 27.11. SEARCH WIZARD GENOME WIDE ASSOCIATION STUDY (GWAS) \TU/ +lmr/m/n/12 281 + [] + +[281] +File: assets/images/search_wizard_genotype_analyses_gwas.png Graphic file (type + bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[282] +File: assets/images/search_wizard_genotype_analyses_manhattan_plot.png Graphic +file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + + +Overfull \hbox (34.74158pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 27.11. SEARCH WIZARD GENOME WIDE ASSOCIATION STUDY (GWAS) \TU/ +lmr/m/n/12 283 + [] + +[283] +File: assets/images/search_wizard_genotype_analyses_qq_plot.png Graphic file (t +ype bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[284] +File: assets/images/waves_breedbase_schema.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[285] +File: assets/images/manage_NIRS_prediction_workflow_intro.png Graphic file (typ +e bmp) + +File: assets/images/manage_NIRS_prediction_workflow_dataset.png Graphic file (t +ype bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[286] +File: assets/images/manage_NIRS_cv.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[287] +File: assets/images/manage_NIRS_snv.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[288] [289] +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[290] +File: assets/images/manage_NIRS_export_model.png Graphic file (type bmp) + +File: assets/images/manage_NIRS_select_model.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[291] +File: assets/images/manage_NIRS_prediction_results.png Graphic file (type bmp) + + +Underfull \hbox (badness 2253) in paragraph at lines 3450--3451 +\TU/lmr/m/n/12 dard normal variate transformation and de-trending of near-infra +red + [] + +[292] [293] +File: assets/images/mixedmodel_tool_model_build_step.png Graphic file (type bmp +) + +[294] +Overfull \hbox (7.04288pt too wide) has occurred while \output is active +\TU/lmr/m/sl/12 27.14. GENOMIC PREDICTION OF CROSS PERFORMANCE (GPCP) \TU/lmr/ +m/n/12 295 + [] + +[295] +File: assets/images/tool_compatibility_details.png Graphic file (type bmp) + +File: assets/images/tool_compatibility_data_summary.png Graphic file (type bmp) + + +[296] [297] [298] +Chapter 28. +File: assets/images/vectorViewer.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[299 + +] +File: assets/images/vectorHighlight.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[300] +File: assets/images/cursorLocation.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[301] +File: assets/images/ZoomIn.png Graphic file (type bmp) + +File: assets/images/ZoomOut.png Graphic file (type bmp) + + +Underfull \vbox (badness 10000) has occurred while \output is active [] + +[302] +File: assets/images/vectorButtons.png Graphic file (type bmp) + +[303] (./BreedbaseManual.aux) + +LaTeX Warning: There were undefined references. + + ) +Here is how much of TeX's memory you used: + 26117 strings out of 478682 + 656962 string characters out of 5849005 + 1860018 words of memory out of 6000000 + 45603 multiletter control sequences out of 15000+600000 + 518740 words of font info for 86 fonts, out of 8000000 for 9000 + 14 hyphenation exceptions out of 8191 + 86i,7n,121p,2343b,499s stack positions out of 10000i,1000n,20000p,200000b,200000s + +Output written on BreedbaseManual.pdf (303 pages). diff --git a/docs/r_markdown_docs/README.Rmd b/docs/r_markdown_docs/README.Rmd new file mode 100644 index 0000000000..9bf138d1cb --- /dev/null +++ b/docs/r_markdown_docs/README.Rmd @@ -0,0 +1,89 @@ +# SGN Documentation +[**View the documentation**](http://solgenomics.github.io/sgn/) + +### Syntax and Use +The documentation is written in [kramdown-flavored](https://kramdown.gettalong.org/) markdown. It also takes advantage of the [Liquid](https://shopify.github.io/liquid/) templating system and the built-in variable in [Jekyll](https://jekyllrb.com), the static site generator that GitHub pages uses. + +This folder (`docs/`) can be used to build the site either via GitHUb pages or via a local Jekyll installation. + +### Guidelines for adding to the documentation + +#### Headers +```markdown +Header 1 OR +=========== + +# Header 1 + +Header 2 OR +----------- + +## Header 2 + +### Header 3 + +#### Header 4 + +##### Header 5 + +###### Header 6 +``` +You probably shouldnt use h1 headers, as the title of the page is always displayed as an h1, and using them elsewhere could be visualy confusing. + +**DONT USE BOLD INSTEAD OF HEADERS** (except in tables). Doing so makes generating a TOC impossible (and also visually looks off.) + +#### Horizontal Rules (Section Seperators) + +In kramdown, a horizontal rule must be preceeded and followed by a blank line: +```markdown +I am above the hr + +----------------- + +I am below the hr +``` + +#### Images/Screenshots + +_For screenshots_: try to make sure that the image is of an unbranded version of the db. + +To insert an image, we need to tell Jekyll to generate the relative path to the file like so +```markdown +![YOUR ALT TEXT]({{'assets/images/YOURIMAGE.png' | relative_url }}) +``` + +#### Links +To insert an link to something outside of the docs, we can use the usual markdown format: +```markdown +[LINK TEXT](http://your.link) +``` +If you want to link to a docs page, use this syntax. Note that we put the **path to the file** (from the `docs` directory), **not the rendered HTML page**, after `link`.): +```markdown +[LINK TEXT]({{ site.baseurl }}{% link your_folder/YOUR_FILE.md %}) +``` +If you want to link to a header on a docs page, we can extend the syntax above like so: +First, we assign the header we are linking to an ID: +```markdown +### I am the header {#your-header-id} +``` +then, we add that ID to the link: +```markdown +[LINK TEXT](#your-header-id) +[LINK TEXT]({{ site.baseurl }}{% link YOUR_FILE.md %}#your-header-id) +``` + +### Creating New Pages +**Every page should start with this YAML "front-matter" before anything else:** +```markdown +--- +title: "YOUR PAGE TITLE HERE" +layout: doc_page +--- +``` +**To insert a table of contents to a page with the following snippet (INCLUDING the comments [excluding them will cause the TOC to be indexed for search and not be readable by the TOC generator of folder pages.]):** +```markdown + +* TOC +{:toc} + +``` diff --git a/docs/r_markdown_docs/README.md b/docs/r_markdown_docs/README.md new file mode 100644 index 0000000000..1ff6214b60 --- /dev/null +++ b/docs/r_markdown_docs/README.md @@ -0,0 +1,162 @@ +## SGN Documentation +[View the documentation](http://solgenomics.github.io/sgn/) + +### Syntax and Use + +This folder `docs/` can be used to build the site through R package - bookdown. More about bookdown here. + +https://bookdown.org/ +https://bookdown.org/yihui/bookdown/ + +The entire page along with the source files in r-markdown format are in the folder +`/docs/r_markdown_docs` + +### Tools. +The best way to work with the documentation is to +1. Install the free R Studio with on your local machine with the sgn repository. +2. Set the working directory environment variable to the path from our documentation R command `setwd("{path_on_local_machine}/sgn/docs/r_markdown_docs")` we can check current working directory by `getwd()`. +3. Install R package “bookdown”, "rmarkdown" and "pandoc". + +### How to work with documentation +1. **Introduction** +This section provides an overview of managing R Markdown documentation using Bookdown and outlines the purpose of this technical manual. +2. **Defining the Document Structure** in `_bookdown.yml` file. +Files: Structure of the document is stored in `_bookdown.yml` file. +`Rmd_files` variable in `_bookdown.yml` is a list of documents used to build a documentation in a given order. If we want to add a new file to documentation - it must be added to the Rmd_files list and a new file must be created in `r_markdown_docs` folder with .Rmd extension. +Setting the Order of Chapters/Sections: Order of chapters is an order of files in `Rmd_files` list. +3. **Adding New Chapters/Sections** +Creating New R Markdown Files: to create a new sectiono or chapter - just create a `.Rmd` file in `/docs/r_markdown_docs` folder. +Updating the Rmd Files List: Once you create a file and want to add it to official documentation update `_bookdown.yml` -> `Rmd_files` with new additions. +4. **Building Your Documentation** +Rendering the Documentation: Best way to render document is to use knit icon ( command ) in `RStudio` when open `Index.Rmd` +Alternative to preview html gitbook format and pdf is to use command in R: +`bookdown::render_book("index.Rmd", "bookdown::pdf_book")` +`bookdown::render_book("index.Rmd", "bookdown::gitbook")` +Previewing the Documentation: Best option is RStudio - with `knit` command we have also a live local server for gitbook html documents. +5. **Deployment and Sharing** +Publishing Your Documentation: + 1. Once changes in documentation are done. Please save all `.Rmd` files in the `r_markdown_docs` folder and check `_bookdown.yml` file if the structure is correct. + 2. Build and check in `RStudio` or any other live server if changes are correct and documentation looks correct. + 3. Commit changes to sgn repository and create a new pull request. With the new pool github through github action will check if there are any changes in `r_markdown_docs` folder and if yes, will trigger a gitaction to automatically build documentation on github containers. + 4. If build action will pass, then when the branch with documation changes is merged with master, Gitaction workflow automatically builds proper gitbook html document, and pdf, and then deploy static version of html to github pages. + That process is completely automated. + + * Build and check locally in RStudio + * Commit changes in `r_markdown_docs` folder and create pull request + * Check if the GitHub Action test for building documentation passes. + * Merge to master and check GitHub Action workflow result. + + +For markdown Gitbook syntax +https://bookdown.org/yihui/rmarkdown/markdown-syntax.html + +# Basic syntax. + +### Emphasis + +*italic* **bold** + +_italic_ __bold__ + +### Headers + +# Header 1 +## Header 2 +### Header 3 + +### Lists + +Unordered List: +* Item 1 +* Item 2 + + Item 2a + + Item 2b + +Ordered List: +1. Item 1 +2. Item 2 +3. Item 3 + + Item 3a + + Item 3b + +### R Code Chunks +R code will be evaluated and printed + +```{r} +summary(cars$dist) +summary(cars$speed) +``` + +### Inline R Code +There were `r nrow(cars)` cars studied + +### Links +Use a plain http address or add a link to a phrase: + +http://example.com + +[linked phrase](http://example.com) + +### Images +Images on the web or local files in the same directory: + +![](http://example.com/logo.png) + +![optional caption text](figures/img.png) + +### Blockquotes +A friend once said: + +> It's always better to give +> than to receive. + +### Plain Code Blocks +Plain code blocks are displayed in a fixed-width font but not evaulated + +``` +This text is displayed verbatim / preformatted +``` +### Inline Code + +We defined the `add` function to compute the sum of two numbers. + +Inline equation: +$equation$ + +Display equation: + +$$ equation $$ + +### Horizontal Rule / Page Break +Three or more asterisks or dashes: + +****** + +------ +Tables +First Header | Second Header +------------- | ------------- +Content Cell | Content Cell +Content Cell | Content Cell +Reference Style Links and Images + +### Links +A [linked phrase][id]. +At the bottom of the document: + +[id]: http://example.com/ "Title" + +### Images +![alt text][id] +At the bottom of the document: +[id]: figures/img.png "Title" + +### Manual Line Breaks +End a line with two or more spaces: + +Roses are red, +Violets are blue. + +### Miscellaneous +superscript^2^ +~~strikethrough~~ \ No newline at end of file diff --git a/docs/r_markdown_docs/_bookdown.yml b/docs/r_markdown_docs/_bookdown.yml new file mode 100644 index 0000000000..ef1ff8058d --- /dev/null +++ b/docs/r_markdown_docs/_bookdown.yml @@ -0,0 +1,40 @@ +rmd_files: [ + "index.Rmd", + "basic_website_usage.Rmd", + "searching_the_database.Rmd", + "managing_user_roles.Rmd", + "managing_breeding_programs.Rmd", + "managing_locations.Rmd", + "managing_accessions.Rmd", + "managing_seedlots.Rmd", + "managing_populations.Rmd", + "managing_crosses.Rmd", + "managing_field_trials.Rmd", + "managing_genotyping_plates.Rmd", + "using_fieldbook_app.Rmd", + "managing_phenotypic_data.Rmd", + "managing_high_dim_phenotyping_data.Rmd", + "managing_barcodes.Rmd", + "using_label_designer.Rmd", + "managing_downloads.Rmd", + "managing_odk_collection.Rmd", + "managing_tissue_samples.Rmd", + "managing_observation_variables.Rmd", + "managing_image_data.Rmd", + "managing_phenotyping_image_data.Rmd", + "managing_vcf_data.Rmd", + "managing_spectral_data.Rmd", + "managing_sequence_metadata.Rmd", + "managing_outliers_in_dataset.Rmd", + "data_analysis_tools.Rmd", + "vector_viewer.Rmd" + ] +book_filename: BreedbaseManual +output_dir: ../ +repo: https://github.com/solgenomics/sgn +delete_merged_file: true +language: + ui: + edit: "Edit" + chapter_name: "Chapter " + diff --git a/docs/r_markdown_docs/_output.yml b/docs/r_markdown_docs/_output.yml new file mode 100644 index 0000000000..166f5a6c5e --- /dev/null +++ b/docs/r_markdown_docs/_output.yml @@ -0,0 +1,32 @@ +bookdown::gitbook: + pandoc_args: ["--lua-filter=conditional-content.lua"] + split_by: chapter + css: assets/style.css + includes: + in_header: hypothesis.html + config: + toc: + collapse: section + before: + after:
  • Published with bookdown
    • + + download: ["pdf"] + sharing: false + info: false +bookdown::pdf_book: + + includes: + in_header: + before_body: + after_body: + keep_tex: yes + fig_crop: false + latex_engine: xelatex + citation_package: natbib + template: null + pandoc_args: [--top-level-division=chapter, --wrap=none, "--lua-filter=conditional-content.lua"] + toc_depth: 3 + toc_unnumbered: no + toc_appendix: yes + highlight_bw: yes + quote_footer: ["\\VA{", "}{}"] diff --git a/docs/r_markdown_docs/assets/gifs/gif1.gif b/docs/r_markdown_docs/assets/gifs/gif1.gif new file mode 100644 index 0000000000..e2a4032837 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif1.gif differ diff --git a/docs/r_markdown_docs/assets/gifs/gif1.png b/docs/r_markdown_docs/assets/gifs/gif1.png new file mode 100644 index 0000000000..35d1de346e Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif1.png differ diff --git a/docs/r_markdown_docs/assets/gifs/gif2.gif b/docs/r_markdown_docs/assets/gifs/gif2.gif new file mode 100644 index 0000000000..8a19226bc3 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif2.gif differ diff --git a/docs/r_markdown_docs/assets/gifs/gif2.png b/docs/r_markdown_docs/assets/gifs/gif2.png new file mode 100644 index 0000000000..e6b237eb31 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif2.png differ diff --git a/docs/r_markdown_docs/assets/gifs/gif3.gif b/docs/r_markdown_docs/assets/gifs/gif3.gif new file mode 100644 index 0000000000..2e57d17af3 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif3.gif differ diff --git a/docs/r_markdown_docs/assets/gifs/gif3.png b/docs/r_markdown_docs/assets/gifs/gif3.png new file mode 100644 index 0000000000..5357e5d44d Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif3.png differ diff --git a/docs/r_markdown_docs/assets/gifs/gif4.gif b/docs/r_markdown_docs/assets/gifs/gif4.gif new file mode 100644 index 0000000000..4391d95306 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif4.gif differ diff --git a/docs/r_markdown_docs/assets/gifs/gif4.png b/docs/r_markdown_docs/assets/gifs/gif4.png new file mode 100644 index 0000000000..badd480621 Binary files /dev/null and b/docs/r_markdown_docs/assets/gifs/gif4.png differ diff --git a/docs/r_markdown_docs/assets/images/Breedbase_HighRes.png b/docs/r_markdown_docs/assets/images/Breedbase_HighRes.png new file mode 100644 index 0000000000..327f8a40aa Binary files /dev/null and b/docs/r_markdown_docs/assets/images/Breedbase_HighRes.png differ diff --git a/docs/r_markdown_docs/assets/images/DatasetHelpImage001.png b/docs/r_markdown_docs/assets/images/DatasetHelpImage001.png new file mode 100644 index 0000000000..39fcadc1b3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/DatasetHelpImage001.png differ diff --git a/docs/r_markdown_docs/assets/images/DatasetHelpImage002.png b/docs/r_markdown_docs/assets/images/DatasetHelpImage002.png new file mode 100644 index 0000000000..a026bb1641 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/DatasetHelpImage002.png differ diff --git a/docs/r_markdown_docs/assets/images/DatasetHelpImage003.png b/docs/r_markdown_docs/assets/images/DatasetHelpImage003.png new file mode 100644 index 0000000000..90021ee69f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/DatasetHelpImage003.png differ diff --git a/docs/r_markdown_docs/assets/images/DatasetHelpImage004.png b/docs/r_markdown_docs/assets/images/DatasetHelpImage004.png new file mode 100644 index 0000000000..190868b7ca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/DatasetHelpImage004.png differ diff --git a/docs/r_markdown_docs/assets/images/DatasetHelpImage005.png b/docs/r_markdown_docs/assets/images/DatasetHelpImage005.png new file mode 100644 index 0000000000..ede4564a33 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/DatasetHelpImage005.png differ diff --git a/docs/r_markdown_docs/assets/images/FB_Section_Header.png b/docs/r_markdown_docs/assets/images/FB_Section_Header.png new file mode 100644 index 0000000000..eb414f98f6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/FB_Section_Header.png differ diff --git a/docs/r_markdown_docs/assets/images/QR_FieldBook_PlayStore.png b/docs/r_markdown_docs/assets/images/QR_FieldBook_PlayStore.png new file mode 100644 index 0000000000..f2f4ea058d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/QR_FieldBook_PlayStore.png differ diff --git a/docs/r_markdown_docs/assets/images/ZoomIn.png b/docs/r_markdown_docs/assets/images/ZoomIn.png new file mode 100644 index 0000000000..6f11f40ede Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ZoomIn.png differ diff --git a/docs/r_markdown_docs/assets/images/ZoomInScreenshot.png b/docs/r_markdown_docs/assets/images/ZoomInScreenshot.png new file mode 100644 index 0000000000..d176b9f456 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ZoomInScreenshot.png differ diff --git a/docs/r_markdown_docs/assets/images/ZoomOut.png b/docs/r_markdown_docs/assets/images/ZoomOut.png new file mode 100644 index 0000000000..1a213c28d4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ZoomOut.png differ diff --git a/docs/r_markdown_docs/assets/images/ZoomOutScreenshot.png b/docs/r_markdown_docs/assets/images/ZoomOutScreenshot.png new file mode 100644 index 0000000000..ec676722e6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ZoomOutScreenshot.png differ diff --git a/docs/r_markdown_docs/assets/images/accession_upload_using_email.png b/docs/r_markdown_docs/assets/images/accession_upload_using_email.png new file mode 100644 index 0000000000..beb797c95d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/accession_upload_using_email.png differ diff --git a/docs/r_markdown_docs/assets/images/add_create_list.png b/docs/r_markdown_docs/assets/images/add_create_list.png new file mode 100644 index 0000000000..ed90d674f3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_create_list.png differ diff --git a/docs/r_markdown_docs/assets/images/add_cross.png b/docs/r_markdown_docs/assets/images/add_cross.png new file mode 100644 index 0000000000..8dffa72fe8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_cross.png differ diff --git a/docs/r_markdown_docs/assets/images/add_cross_info.png b/docs/r_markdown_docs/assets/images/add_cross_info.png new file mode 100644 index 0000000000..31b4a0991d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_cross_info.png differ diff --git a/docs/r_markdown_docs/assets/images/add_crossingtrial.png b/docs/r_markdown_docs/assets/images/add_crossingtrial.png new file mode 100644 index 0000000000..5b2253d305 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_crossingtrial.png differ diff --git a/docs/r_markdown_docs/assets/images/add_management_factor_dialog.png b/docs/r_markdown_docs/assets/images/add_management_factor_dialog.png new file mode 100644 index 0000000000..173817d2df Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_management_factor_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/add_management_factor_name_dialog.png b/docs/r_markdown_docs/assets/images/add_management_factor_name_dialog.png new file mode 100644 index 0000000000..de31941a46 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_management_factor_name_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/add_transplanting_date.png b/docs/r_markdown_docs/assets/images/add_transplanting_date.png new file mode 100644 index 0000000000..6f352b79d8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/add_transplanting_date.png differ diff --git a/docs/r_markdown_docs/assets/images/anova-dm.png b/docs/r_markdown_docs/assets/images/anova-dm.png new file mode 100644 index 0000000000..6fab8002f8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/anova-dm.png differ diff --git a/docs/r_markdown_docs/assets/images/breeding_programs_screenshot.png b/docs/r_markdown_docs/assets/images/breeding_programs_screenshot.png new file mode 100644 index 0000000000..370b8e77a8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/breeding_programs_screenshot.png differ diff --git a/docs/r_markdown_docs/assets/images/create_phenotype_spreadsheet_dialog.png b/docs/r_markdown_docs/assets/images/create_phenotype_spreadsheet_dialog.png new file mode 100644 index 0000000000..56cee006bf Binary files /dev/null and b/docs/r_markdown_docs/assets/images/create_phenotype_spreadsheet_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/create_transcriptomics_protocol.png b/docs/r_markdown_docs/assets/images/create_transcriptomics_protocol.png new file mode 100644 index 0000000000..bfb0c98061 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/create_transcriptomics_protocol.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_add_progenies.png b/docs/r_markdown_docs/assets/images/cross_add_progenies.png new file mode 100644 index 0000000000..5f2c6d7f17 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_add_progenies.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_page_1.png b/docs/r_markdown_docs/assets/images/cross_page_1.png new file mode 100644 index 0000000000..163cf9b8c2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_page_1.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_page_2.png b/docs/r_markdown_docs/assets/images/cross_page_2.png new file mode 100644 index 0000000000..6322ce450a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_page_2.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_upload_format.png b/docs/r_markdown_docs/assets/images/cross_upload_format.png new file mode 100644 index 0000000000..d20f2de7a8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_upload_format.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_upload_page.png b/docs/r_markdown_docs/assets/images/cross_upload_page.png new file mode 100644 index 0000000000..cd2aec698a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_upload_page.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_initial_dialog.png b/docs/r_markdown_docs/assets/images/cross_wishlist_initial_dialog.png new file mode 100644 index 0000000000..ee1a876a55 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_initial_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_01.png b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_01.png new file mode 100644 index 0000000000..9e17f10560 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_01.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_02.png b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_02.png new file mode 100644 index 0000000000..d7113566a6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_02.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_03.png b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_03.png new file mode 100644 index 0000000000..38f1448719 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_03.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_04.png b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_04.png new file mode 100644 index 0000000000..9078cf0224 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_not_using_list_04.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_01.png b/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_01.png new file mode 100644 index 0000000000..36c8ee2a0a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_01.png differ diff --git a/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_02.png b/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_02.png new file mode 100644 index 0000000000..4c656f9c93 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cross_wishlist_using_list_02.png differ diff --git a/docs/r_markdown_docs/assets/images/crossinfo_upload_spreadsheet.png b/docs/r_markdown_docs/assets/images/crossinfo_upload_spreadsheet.png new file mode 100644 index 0000000000..05c4d73197 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/crossinfo_upload_spreadsheet.png differ diff --git a/docs/r_markdown_docs/assets/images/crossingtrial_1.png b/docs/r_markdown_docs/assets/images/crossingtrial_1.png new file mode 100644 index 0000000000..d0891ffd66 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/crossingtrial_1.png differ diff --git a/docs/r_markdown_docs/assets/images/crossingtrial_2.png b/docs/r_markdown_docs/assets/images/crossingtrial_2.png new file mode 100644 index 0000000000..60bc304c48 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/crossingtrial_2.png differ diff --git a/docs/r_markdown_docs/assets/images/cursorLocation.png b/docs/r_markdown_docs/assets/images/cursorLocation.png new file mode 100644 index 0000000000..1ddde75b7c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/cursorLocation.png differ diff --git a/docs/r_markdown_docs/assets/images/data_matrix_example.png b/docs/r_markdown_docs/assets/images/data_matrix_example.png new file mode 100644 index 0000000000..a6dd103b73 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/data_matrix_example.png differ diff --git a/docs/r_markdown_docs/assets/images/data_matrix_headers.png b/docs/r_markdown_docs/assets/images/data_matrix_headers.png new file mode 100644 index 0000000000..01673de8a0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/data_matrix_headers.png differ diff --git a/docs/r_markdown_docs/assets/images/download_transcriptomics.png b/docs/r_markdown_docs/assets/images/download_transcriptomics.png new file mode 100644 index 0000000000..1fb6eb1421 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/download_transcriptomics.png differ diff --git a/docs/r_markdown_docs/assets/images/edit_cross_info.png b/docs/r_markdown_docs/assets/images/edit_cross_info.png new file mode 100644 index 0000000000..299012994c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/edit_cross_info.png differ diff --git a/docs/r_markdown_docs/assets/images/field-book-icon.png b/docs/r_markdown_docs/assets/images/field-book-icon.png new file mode 100644 index 0000000000..ea84293bd5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/field-book-icon.png differ diff --git a/docs/r_markdown_docs/assets/images/field_map_download_dialog.png b/docs/r_markdown_docs/assets/images/field_map_download_dialog.png new file mode 100644 index 0000000000..d2db3219a5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/field_map_download_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/field_map_download_link.png b/docs/r_markdown_docs/assets/images/field_map_download_link.png new file mode 100644 index 0000000000..74b4d1f84b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/field_map_download_link.png differ diff --git a/docs/r_markdown_docs/assets/images/field_map_download_spreadsheet.png b/docs/r_markdown_docs/assets/images/field_map_download_spreadsheet.png new file mode 100644 index 0000000000..301a967bf0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/field_map_download_spreadsheet.png differ diff --git a/docs/r_markdown_docs/assets/images/fieldmap_display_plot_image.png b/docs/r_markdown_docs/assets/images/fieldmap_display_plot_image.png new file mode 100644 index 0000000000..ae0e62050c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/fieldmap_display_plot_image.png differ diff --git a/docs/r_markdown_docs/assets/images/fieldmap_multi_trial_layout.png b/docs/r_markdown_docs/assets/images/fieldmap_multi_trial_layout.png new file mode 100644 index 0000000000..e25d6a62ac Binary files /dev/null and b/docs/r_markdown_docs/assets/images/fieldmap_multi_trial_layout.png differ diff --git a/docs/r_markdown_docs/assets/images/fieldmap_plot_image.png b/docs/r_markdown_docs/assets/images/fieldmap_plot_image.png new file mode 100644 index 0000000000..810618f60a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/fieldmap_plot_image.png differ diff --git a/docs/r_markdown_docs/assets/images/fieldmap_trial_layout.png b/docs/r_markdown_docs/assets/images/fieldmap_trial_layout.png new file mode 100644 index 0000000000..7e9ee21cfd Binary files /dev/null and b/docs/r_markdown_docs/assets/images/fieldmap_trial_layout.png differ diff --git a/docs/r_markdown_docs/assets/images/fieldmap_view_plot_image.png b/docs/r_markdown_docs/assets/images/fieldmap_view_plot_image.png new file mode 100644 index 0000000000..97eebb3ea3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/fieldmap_view_plot_image.png differ diff --git a/docs/r_markdown_docs/assets/images/filename_verify_success.png b/docs/r_markdown_docs/assets/images/filename_verify_success.png new file mode 100644 index 0000000000..001eff461c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/filename_verify_success.png differ diff --git a/docs/r_markdown_docs/assets/images/genetic-gain.png b/docs/r_markdown_docs/assets/images/genetic-gain.png new file mode 100644 index 0000000000..b03ac03425 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/genetic-gain.png differ diff --git a/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_delete.png b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_delete.png new file mode 100644 index 0000000000..95a7badd4c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_delete.png differ diff --git a/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_markers.png b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_markers.png new file mode 100644 index 0000000000..eaa847ffbd Binary files /dev/null and b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_markers.png differ diff --git a/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_samples.png b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_samples.png new file mode 100644 index 0000000000..2610bd38dc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/genotyping_protocol_detail_page_samples.png differ diff --git a/docs/r_markdown_docs/assets/images/genotyping_protocols_search.png b/docs/r_markdown_docs/assets/images/genotyping_protocols_search.png new file mode 100644 index 0000000000..306ee9853b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/genotyping_protocols_search.png differ diff --git a/docs/r_markdown_docs/assets/images/hclustering.png b/docs/r_markdown_docs/assets/images/hclustering.png new file mode 100644 index 0000000000..283b1653be Binary files /dev/null and b/docs/r_markdown_docs/assets/images/hclustering.png differ diff --git a/docs/r_markdown_docs/assets/images/heatmap_assayed_trait_view.png b/docs/r_markdown_docs/assets/images/heatmap_assayed_trait_view.png new file mode 100644 index 0000000000..97b2ee8865 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/heatmap_assayed_trait_view.png differ diff --git a/docs/r_markdown_docs/assets/images/heatmap_default_display.png b/docs/r_markdown_docs/assets/images/heatmap_default_display.png new file mode 100644 index 0000000000..cc2ce7daa1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/heatmap_default_display.png differ diff --git a/docs/r_markdown_docs/assets/images/heatmap_spatial_correction_button.png b/docs/r_markdown_docs/assets/images/heatmap_spatial_correction_button.png new file mode 100644 index 0000000000..adc85fd6aa Binary files /dev/null and b/docs/r_markdown_docs/assets/images/heatmap_spatial_correction_button.png differ diff --git a/docs/r_markdown_docs/assets/images/image112.png b/docs/r_markdown_docs/assets/images/image112.png new file mode 100755 index 0000000000..0b80957ce9 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image112.png differ diff --git a/docs/r_markdown_docs/assets/images/image114.png b/docs/r_markdown_docs/assets/images/image114.png new file mode 100755 index 0000000000..746181e110 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image114.png differ diff --git a/docs/r_markdown_docs/assets/images/image115.png b/docs/r_markdown_docs/assets/images/image115.png new file mode 100755 index 0000000000..419b5e10ae Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image115.png differ diff --git a/docs/r_markdown_docs/assets/images/image116.png b/docs/r_markdown_docs/assets/images/image116.png new file mode 100755 index 0000000000..fce3a0c594 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image116.png differ diff --git a/docs/r_markdown_docs/assets/images/image12.png b/docs/r_markdown_docs/assets/images/image12.png new file mode 100755 index 0000000000..1bac7db039 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image12.png differ diff --git a/docs/r_markdown_docs/assets/images/image131.png b/docs/r_markdown_docs/assets/images/image131.png new file mode 100755 index 0000000000..e6e53d9352 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image131.png differ diff --git a/docs/r_markdown_docs/assets/images/image133.png b/docs/r_markdown_docs/assets/images/image133.png new file mode 100755 index 0000000000..82d88ccc5f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image133.png differ diff --git a/docs/r_markdown_docs/assets/images/image134.png b/docs/r_markdown_docs/assets/images/image134.png new file mode 100755 index 0000000000..95c7a62740 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image134.png differ diff --git a/docs/r_markdown_docs/assets/images/image135.png b/docs/r_markdown_docs/assets/images/image135.png new file mode 100755 index 0000000000..1a9bc7698c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image135.png differ diff --git a/docs/r_markdown_docs/assets/images/image136.png b/docs/r_markdown_docs/assets/images/image136.png new file mode 100755 index 0000000000..cecfe15ed7 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image136.png differ diff --git a/docs/r_markdown_docs/assets/images/image138.png b/docs/r_markdown_docs/assets/images/image138.png new file mode 100755 index 0000000000..6ef8fe51c0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image138.png differ diff --git a/docs/r_markdown_docs/assets/images/image140.png b/docs/r_markdown_docs/assets/images/image140.png new file mode 100755 index 0000000000..4498435d00 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image140.png differ diff --git a/docs/r_markdown_docs/assets/images/image141.png b/docs/r_markdown_docs/assets/images/image141.png new file mode 100755 index 0000000000..705f69139b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image141.png differ diff --git a/docs/r_markdown_docs/assets/images/image142.png b/docs/r_markdown_docs/assets/images/image142.png new file mode 100755 index 0000000000..ba8bb65557 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image142.png differ diff --git a/docs/r_markdown_docs/assets/images/image144.png b/docs/r_markdown_docs/assets/images/image144.png new file mode 100755 index 0000000000..f61d399c2c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image144.png differ diff --git a/docs/r_markdown_docs/assets/images/image146.png b/docs/r_markdown_docs/assets/images/image146.png new file mode 100755 index 0000000000..7cc666215d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image146.png differ diff --git a/docs/r_markdown_docs/assets/images/image147.png b/docs/r_markdown_docs/assets/images/image147.png new file mode 100755 index 0000000000..e9dcc6562e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image147.png differ diff --git a/docs/r_markdown_docs/assets/images/image148.png b/docs/r_markdown_docs/assets/images/image148.png new file mode 100755 index 0000000000..de1cf05066 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image148.png differ diff --git a/docs/r_markdown_docs/assets/images/image149.png b/docs/r_markdown_docs/assets/images/image149.png new file mode 100755 index 0000000000..2266b03e09 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image149.png differ diff --git a/docs/r_markdown_docs/assets/images/image150.png b/docs/r_markdown_docs/assets/images/image150.png new file mode 100755 index 0000000000..7bbd333b97 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image150.png differ diff --git a/docs/r_markdown_docs/assets/images/image151.png b/docs/r_markdown_docs/assets/images/image151.png new file mode 100755 index 0000000000..4fb1e7ce89 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image151.png differ diff --git a/docs/r_markdown_docs/assets/images/image152.png b/docs/r_markdown_docs/assets/images/image152.png new file mode 100755 index 0000000000..17c9320c04 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image152.png differ diff --git a/docs/r_markdown_docs/assets/images/image153.png b/docs/r_markdown_docs/assets/images/image153.png new file mode 100755 index 0000000000..6d73dc3465 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image153.png differ diff --git a/docs/r_markdown_docs/assets/images/image154.png b/docs/r_markdown_docs/assets/images/image154.png new file mode 100755 index 0000000000..ea4c6bf41c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image154.png differ diff --git a/docs/r_markdown_docs/assets/images/image155.png b/docs/r_markdown_docs/assets/images/image155.png new file mode 100755 index 0000000000..f286c61bec Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image155.png differ diff --git a/docs/r_markdown_docs/assets/images/image156.png b/docs/r_markdown_docs/assets/images/image156.png new file mode 100755 index 0000000000..1c40ee4de8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image156.png differ diff --git a/docs/r_markdown_docs/assets/images/image157.png b/docs/r_markdown_docs/assets/images/image157.png new file mode 100755 index 0000000000..46add37382 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image157.png differ diff --git a/docs/r_markdown_docs/assets/images/image158.png b/docs/r_markdown_docs/assets/images/image158.png new file mode 100755 index 0000000000..f4d62347c0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image158.png differ diff --git a/docs/r_markdown_docs/assets/images/image16.png b/docs/r_markdown_docs/assets/images/image16.png new file mode 100755 index 0000000000..dd9562b258 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image16.png differ diff --git a/docs/r_markdown_docs/assets/images/image160.png b/docs/r_markdown_docs/assets/images/image160.png new file mode 100755 index 0000000000..4cbd495334 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image160.png differ diff --git a/docs/r_markdown_docs/assets/images/image161.png b/docs/r_markdown_docs/assets/images/image161.png new file mode 100644 index 0000000000..1c1bc1c5f8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image161.png differ diff --git a/docs/r_markdown_docs/assets/images/image162.png b/docs/r_markdown_docs/assets/images/image162.png new file mode 100755 index 0000000000..5a0ba99f6e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image162.png differ diff --git a/docs/r_markdown_docs/assets/images/image163.png b/docs/r_markdown_docs/assets/images/image163.png new file mode 100755 index 0000000000..83fb9cfe59 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image163.png differ diff --git a/docs/r_markdown_docs/assets/images/image165.png b/docs/r_markdown_docs/assets/images/image165.png new file mode 100755 index 0000000000..373c6ff5bb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image165.png differ diff --git a/docs/r_markdown_docs/assets/images/image166.png b/docs/r_markdown_docs/assets/images/image166.png new file mode 100755 index 0000000000..f71291056c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image166.png differ diff --git a/docs/r_markdown_docs/assets/images/image170.png b/docs/r_markdown_docs/assets/images/image170.png new file mode 100755 index 0000000000..82d88ccc5f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image170.png differ diff --git a/docs/r_markdown_docs/assets/images/image171.png b/docs/r_markdown_docs/assets/images/image171.png new file mode 100755 index 0000000000..7a54c60577 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image171.png differ diff --git a/docs/r_markdown_docs/assets/images/image173.png b/docs/r_markdown_docs/assets/images/image173.png new file mode 100755 index 0000000000..6ae4bbeba3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image173.png differ diff --git a/docs/r_markdown_docs/assets/images/image174.png b/docs/r_markdown_docs/assets/images/image174.png new file mode 100755 index 0000000000..94140f5086 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image174.png differ diff --git a/docs/r_markdown_docs/assets/images/image175.png b/docs/r_markdown_docs/assets/images/image175.png new file mode 100755 index 0000000000..998fa01279 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image175.png differ diff --git a/docs/r_markdown_docs/assets/images/image177.png b/docs/r_markdown_docs/assets/images/image177.png new file mode 100755 index 0000000000..d18b19555a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image177.png differ diff --git a/docs/r_markdown_docs/assets/images/image178.png b/docs/r_markdown_docs/assets/images/image178.png new file mode 100755 index 0000000000..1642648821 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image178.png differ diff --git a/docs/r_markdown_docs/assets/images/image181.png b/docs/r_markdown_docs/assets/images/image181.png new file mode 100755 index 0000000000..b8f0dbfecb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image181.png differ diff --git a/docs/r_markdown_docs/assets/images/image182.png b/docs/r_markdown_docs/assets/images/image182.png new file mode 100755 index 0000000000..d1b7c82b8a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image182.png differ diff --git a/docs/r_markdown_docs/assets/images/image183.png b/docs/r_markdown_docs/assets/images/image183.png new file mode 100644 index 0000000000..43cd8786ea Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image183.png differ diff --git a/docs/r_markdown_docs/assets/images/image184.png b/docs/r_markdown_docs/assets/images/image184.png new file mode 100755 index 0000000000..12a7d7d721 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image184.png differ diff --git a/docs/r_markdown_docs/assets/images/image185.png b/docs/r_markdown_docs/assets/images/image185.png new file mode 100755 index 0000000000..97f780f01c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image185.png differ diff --git a/docs/r_markdown_docs/assets/images/image187.png b/docs/r_markdown_docs/assets/images/image187.png new file mode 100755 index 0000000000..3f3a75cb86 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image187.png differ diff --git a/docs/r_markdown_docs/assets/images/image189.png b/docs/r_markdown_docs/assets/images/image189.png new file mode 100755 index 0000000000..12991b499c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image189.png differ diff --git a/docs/r_markdown_docs/assets/images/image190.png b/docs/r_markdown_docs/assets/images/image190.png new file mode 100755 index 0000000000..5291fa3849 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image190.png differ diff --git a/docs/r_markdown_docs/assets/images/image20.png b/docs/r_markdown_docs/assets/images/image20.png new file mode 100755 index 0000000000..499143eccc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image20.png differ diff --git a/docs/r_markdown_docs/assets/images/image21.png b/docs/r_markdown_docs/assets/images/image21.png new file mode 100755 index 0000000000..914ce3e30e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image21.png differ diff --git a/docs/r_markdown_docs/assets/images/image22.png b/docs/r_markdown_docs/assets/images/image22.png new file mode 100755 index 0000000000..8819e07eda Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image22.png differ diff --git a/docs/r_markdown_docs/assets/images/image243.png b/docs/r_markdown_docs/assets/images/image243.png new file mode 100755 index 0000000000..6d66494469 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image243.png differ diff --git a/docs/r_markdown_docs/assets/images/image244.png b/docs/r_markdown_docs/assets/images/image244.png new file mode 100755 index 0000000000..7f9dabe127 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image244.png differ diff --git a/docs/r_markdown_docs/assets/images/image245.png b/docs/r_markdown_docs/assets/images/image245.png new file mode 100755 index 0000000000..c06cc28436 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image245.png differ diff --git a/docs/r_markdown_docs/assets/images/image246.png b/docs/r_markdown_docs/assets/images/image246.png new file mode 100755 index 0000000000..2bd5719702 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image246.png differ diff --git a/docs/r_markdown_docs/assets/images/image247.png b/docs/r_markdown_docs/assets/images/image247.png new file mode 100755 index 0000000000..3c0e86d875 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image247.png differ diff --git a/docs/r_markdown_docs/assets/images/image248.png b/docs/r_markdown_docs/assets/images/image248.png new file mode 100755 index 0000000000..1eb3efa6d5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image248.png differ diff --git a/docs/r_markdown_docs/assets/images/image249.png b/docs/r_markdown_docs/assets/images/image249.png new file mode 100755 index 0000000000..1f31b1eca8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image249.png differ diff --git a/docs/r_markdown_docs/assets/images/image251.png b/docs/r_markdown_docs/assets/images/image251.png new file mode 100755 index 0000000000..ce3614e8ca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image251.png differ diff --git a/docs/r_markdown_docs/assets/images/image252.png b/docs/r_markdown_docs/assets/images/image252.png new file mode 100755 index 0000000000..35e2d351b2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image252.png differ diff --git a/docs/r_markdown_docs/assets/images/image253.png b/docs/r_markdown_docs/assets/images/image253.png new file mode 100755 index 0000000000..5abd21b6c0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image253.png differ diff --git a/docs/r_markdown_docs/assets/images/image254.png b/docs/r_markdown_docs/assets/images/image254.png new file mode 100755 index 0000000000..a0afa9c542 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image254.png differ diff --git a/docs/r_markdown_docs/assets/images/image255.png b/docs/r_markdown_docs/assets/images/image255.png new file mode 100755 index 0000000000..af493d613f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image255.png differ diff --git a/docs/r_markdown_docs/assets/images/image256.png b/docs/r_markdown_docs/assets/images/image256.png new file mode 100755 index 0000000000..47cd197157 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image256.png differ diff --git a/docs/r_markdown_docs/assets/images/image257.png b/docs/r_markdown_docs/assets/images/image257.png new file mode 100755 index 0000000000..a90e411b41 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image257.png differ diff --git a/docs/r_markdown_docs/assets/images/image258.png b/docs/r_markdown_docs/assets/images/image258.png new file mode 100755 index 0000000000..ea6e94218f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image258.png differ diff --git a/docs/r_markdown_docs/assets/images/image261.png b/docs/r_markdown_docs/assets/images/image261.png new file mode 100755 index 0000000000..e409ac7a54 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image261.png differ diff --git a/docs/r_markdown_docs/assets/images/image262.png b/docs/r_markdown_docs/assets/images/image262.png new file mode 100755 index 0000000000..aa8e40544b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image262.png differ diff --git a/docs/r_markdown_docs/assets/images/image263.png b/docs/r_markdown_docs/assets/images/image263.png new file mode 100755 index 0000000000..fa255d7210 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image263.png differ diff --git a/docs/r_markdown_docs/assets/images/image264.png b/docs/r_markdown_docs/assets/images/image264.png new file mode 100755 index 0000000000..7f92ca86fe Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image264.png differ diff --git a/docs/r_markdown_docs/assets/images/image265.png b/docs/r_markdown_docs/assets/images/image265.png new file mode 100755 index 0000000000..f46b69fe61 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image265.png differ diff --git a/docs/r_markdown_docs/assets/images/image267.png b/docs/r_markdown_docs/assets/images/image267.png new file mode 100755 index 0000000000..bedc89d416 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image267.png differ diff --git a/docs/r_markdown_docs/assets/images/image268.png b/docs/r_markdown_docs/assets/images/image268.png new file mode 100755 index 0000000000..c3663cc645 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image268.png differ diff --git a/docs/r_markdown_docs/assets/images/image269.png b/docs/r_markdown_docs/assets/images/image269.png new file mode 100755 index 0000000000..13b7a4dc7e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image269.png differ diff --git a/docs/r_markdown_docs/assets/images/image270.png b/docs/r_markdown_docs/assets/images/image270.png new file mode 100755 index 0000000000..3561968d63 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image270.png differ diff --git a/docs/r_markdown_docs/assets/images/image273.png b/docs/r_markdown_docs/assets/images/image273.png new file mode 100755 index 0000000000..53d094c342 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image273.png differ diff --git a/docs/r_markdown_docs/assets/images/image274.png b/docs/r_markdown_docs/assets/images/image274.png new file mode 100755 index 0000000000..0a5e431368 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image274.png differ diff --git a/docs/r_markdown_docs/assets/images/image276.png b/docs/r_markdown_docs/assets/images/image276.png new file mode 100755 index 0000000000..0180709304 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image276.png differ diff --git a/docs/r_markdown_docs/assets/images/image277.png b/docs/r_markdown_docs/assets/images/image277.png new file mode 100755 index 0000000000..e9b92546dc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image277.png differ diff --git a/docs/r_markdown_docs/assets/images/image279.png b/docs/r_markdown_docs/assets/images/image279.png new file mode 100755 index 0000000000..f7142bda01 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image279.png differ diff --git a/docs/r_markdown_docs/assets/images/image281.png b/docs/r_markdown_docs/assets/images/image281.png new file mode 100755 index 0000000000..ee7d2bfd03 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image281.png differ diff --git a/docs/r_markdown_docs/assets/images/image282.png b/docs/r_markdown_docs/assets/images/image282.png new file mode 100755 index 0000000000..f04a0b19ed Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image282.png differ diff --git a/docs/r_markdown_docs/assets/images/image285.png b/docs/r_markdown_docs/assets/images/image285.png new file mode 100755 index 0000000000..762cb603be Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image285.png differ diff --git a/docs/r_markdown_docs/assets/images/image286.png b/docs/r_markdown_docs/assets/images/image286.png new file mode 100755 index 0000000000..9508584286 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image286.png differ diff --git a/docs/r_markdown_docs/assets/images/image287.png b/docs/r_markdown_docs/assets/images/image287.png new file mode 100755 index 0000000000..ec380e7aca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image287.png differ diff --git a/docs/r_markdown_docs/assets/images/image288.png b/docs/r_markdown_docs/assets/images/image288.png new file mode 100755 index 0000000000..f88361de94 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image288.png differ diff --git a/docs/r_markdown_docs/assets/images/image290.png b/docs/r_markdown_docs/assets/images/image290.png new file mode 100755 index 0000000000..e32886df83 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image290.png differ diff --git a/docs/r_markdown_docs/assets/images/image295.png b/docs/r_markdown_docs/assets/images/image295.png new file mode 100755 index 0000000000..da3649ee0e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image295.png differ diff --git a/docs/r_markdown_docs/assets/images/image296.png b/docs/r_markdown_docs/assets/images/image296.png new file mode 100755 index 0000000000..954ef7ce0f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image296.png differ diff --git a/docs/r_markdown_docs/assets/images/image297.png b/docs/r_markdown_docs/assets/images/image297.png new file mode 100755 index 0000000000..2c52987ac6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image297.png differ diff --git a/docs/r_markdown_docs/assets/images/image298.png b/docs/r_markdown_docs/assets/images/image298.png new file mode 100755 index 0000000000..5b6c907add Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image298.png differ diff --git a/docs/r_markdown_docs/assets/images/image302.png b/docs/r_markdown_docs/assets/images/image302.png new file mode 100755 index 0000000000..82d88ccc5f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image302.png differ diff --git a/docs/r_markdown_docs/assets/images/image303.png b/docs/r_markdown_docs/assets/images/image303.png new file mode 100755 index 0000000000..aa9a47d304 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image303.png differ diff --git a/docs/r_markdown_docs/assets/images/image304.png b/docs/r_markdown_docs/assets/images/image304.png new file mode 100755 index 0000000000..20b6a7f87a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image304.png differ diff --git a/docs/r_markdown_docs/assets/images/image314.png b/docs/r_markdown_docs/assets/images/image314.png new file mode 100755 index 0000000000..1795799bca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image314.png differ diff --git a/docs/r_markdown_docs/assets/images/image317.png b/docs/r_markdown_docs/assets/images/image317.png new file mode 100755 index 0000000000..24f34d1212 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image317.png differ diff --git a/docs/r_markdown_docs/assets/images/image318.png b/docs/r_markdown_docs/assets/images/image318.png new file mode 100755 index 0000000000..7994cc76c4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image318.png differ diff --git a/docs/r_markdown_docs/assets/images/image319.png b/docs/r_markdown_docs/assets/images/image319.png new file mode 100755 index 0000000000..7978553330 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image319.png differ diff --git a/docs/r_markdown_docs/assets/images/image320.png b/docs/r_markdown_docs/assets/images/image320.png new file mode 100755 index 0000000000..328483ccf1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image320.png differ diff --git a/docs/r_markdown_docs/assets/images/image321.png b/docs/r_markdown_docs/assets/images/image321.png new file mode 100755 index 0000000000..225349d2d1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image321.png differ diff --git a/docs/r_markdown_docs/assets/images/image322.png b/docs/r_markdown_docs/assets/images/image322.png new file mode 100755 index 0000000000..c0810dd9af Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image322.png differ diff --git a/docs/r_markdown_docs/assets/images/image324.png b/docs/r_markdown_docs/assets/images/image324.png new file mode 100755 index 0000000000..8d0a69b00c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image324.png differ diff --git a/docs/r_markdown_docs/assets/images/image325.png b/docs/r_markdown_docs/assets/images/image325.png new file mode 100755 index 0000000000..4e5fef3726 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image325.png differ diff --git a/docs/r_markdown_docs/assets/images/image326.png b/docs/r_markdown_docs/assets/images/image326.png new file mode 100755 index 0000000000..c74de20ea8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image326.png differ diff --git a/docs/r_markdown_docs/assets/images/image327.png b/docs/r_markdown_docs/assets/images/image327.png new file mode 100755 index 0000000000..b00a978397 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image327.png differ diff --git a/docs/r_markdown_docs/assets/images/image328.png b/docs/r_markdown_docs/assets/images/image328.png new file mode 100755 index 0000000000..8f4cb6e69f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image328.png differ diff --git a/docs/r_markdown_docs/assets/images/image329.png b/docs/r_markdown_docs/assets/images/image329.png new file mode 100755 index 0000000000..69053efe9b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image329.png differ diff --git a/docs/r_markdown_docs/assets/images/image330.png b/docs/r_markdown_docs/assets/images/image330.png new file mode 100755 index 0000000000..95ac06fcf5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image330.png differ diff --git a/docs/r_markdown_docs/assets/images/image331.png b/docs/r_markdown_docs/assets/images/image331.png new file mode 100755 index 0000000000..dca1dd4e9d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image331.png differ diff --git a/docs/r_markdown_docs/assets/images/image332.png b/docs/r_markdown_docs/assets/images/image332.png new file mode 100644 index 0000000000..4d823cc22d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image332.png differ diff --git a/docs/r_markdown_docs/assets/images/image333.png b/docs/r_markdown_docs/assets/images/image333.png new file mode 100755 index 0000000000..95498cefdb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image333.png differ diff --git a/docs/r_markdown_docs/assets/images/image334.png b/docs/r_markdown_docs/assets/images/image334.png new file mode 100755 index 0000000000..3c5e6a4250 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image334.png differ diff --git a/docs/r_markdown_docs/assets/images/image336.png b/docs/r_markdown_docs/assets/images/image336.png new file mode 100755 index 0000000000..d44abab00b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image336.png differ diff --git a/docs/r_markdown_docs/assets/images/image337.png b/docs/r_markdown_docs/assets/images/image337.png new file mode 100644 index 0000000000..5019100cee Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image337.png differ diff --git a/docs/r_markdown_docs/assets/images/image338.png b/docs/r_markdown_docs/assets/images/image338.png new file mode 100755 index 0000000000..234ce440a8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image338.png differ diff --git a/docs/r_markdown_docs/assets/images/image340.png b/docs/r_markdown_docs/assets/images/image340.png new file mode 100755 index 0000000000..f187b051a2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image340.png differ diff --git a/docs/r_markdown_docs/assets/images/image341.png b/docs/r_markdown_docs/assets/images/image341.png new file mode 100755 index 0000000000..264cb8cc7c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image341.png differ diff --git a/docs/r_markdown_docs/assets/images/image342.png b/docs/r_markdown_docs/assets/images/image342.png new file mode 100755 index 0000000000..4511681336 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image342.png differ diff --git a/docs/r_markdown_docs/assets/images/image343.png b/docs/r_markdown_docs/assets/images/image343.png new file mode 100755 index 0000000000..30dba1abbb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image343.png differ diff --git a/docs/r_markdown_docs/assets/images/image344.png b/docs/r_markdown_docs/assets/images/image344.png new file mode 100755 index 0000000000..a4f3b0989e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image344.png differ diff --git a/docs/r_markdown_docs/assets/images/image345.png b/docs/r_markdown_docs/assets/images/image345.png new file mode 100755 index 0000000000..e3c149a6c3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image345.png differ diff --git a/docs/r_markdown_docs/assets/images/image346.png b/docs/r_markdown_docs/assets/images/image346.png new file mode 100755 index 0000000000..c4a917a29f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image346.png differ diff --git a/docs/r_markdown_docs/assets/images/image347.png b/docs/r_markdown_docs/assets/images/image347.png new file mode 100644 index 0000000000..6758585949 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image347.png differ diff --git a/docs/r_markdown_docs/assets/images/image348.png b/docs/r_markdown_docs/assets/images/image348.png new file mode 100644 index 0000000000..cc815e4e0e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image348.png differ diff --git a/docs/r_markdown_docs/assets/images/image68.png b/docs/r_markdown_docs/assets/images/image68.png new file mode 100755 index 0000000000..7073a452d6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image68.png differ diff --git a/docs/r_markdown_docs/assets/images/image69.png b/docs/r_markdown_docs/assets/images/image69.png new file mode 100755 index 0000000000..c0810dd9af Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image69.png differ diff --git a/docs/r_markdown_docs/assets/images/image71.png b/docs/r_markdown_docs/assets/images/image71.png new file mode 100755 index 0000000000..ae11221bb4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image71.png differ diff --git a/docs/r_markdown_docs/assets/images/image72.png b/docs/r_markdown_docs/assets/images/image72.png new file mode 100755 index 0000000000..b49869af5c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image72.png differ diff --git a/docs/r_markdown_docs/assets/images/image74.png b/docs/r_markdown_docs/assets/images/image74.png new file mode 100755 index 0000000000..46e6017f0e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image74.png differ diff --git a/docs/r_markdown_docs/assets/images/image75.png b/docs/r_markdown_docs/assets/images/image75.png new file mode 100755 index 0000000000..c844460a0c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image75.png differ diff --git a/docs/r_markdown_docs/assets/images/image76.png b/docs/r_markdown_docs/assets/images/image76.png new file mode 100755 index 0000000000..d938c39e5d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image76.png differ diff --git a/docs/r_markdown_docs/assets/images/image78.png b/docs/r_markdown_docs/assets/images/image78.png new file mode 100755 index 0000000000..a4dc9e6860 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image78.png differ diff --git a/docs/r_markdown_docs/assets/images/image79.png b/docs/r_markdown_docs/assets/images/image79.png new file mode 100755 index 0000000000..8ff809847d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image79.png differ diff --git a/docs/r_markdown_docs/assets/images/image81.png b/docs/r_markdown_docs/assets/images/image81.png new file mode 100755 index 0000000000..8b220cb666 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image81.png differ diff --git a/docs/r_markdown_docs/assets/images/image82.png b/docs/r_markdown_docs/assets/images/image82.png new file mode 100755 index 0000000000..f65c59fa97 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image82.png differ diff --git a/docs/r_markdown_docs/assets/images/image83.png b/docs/r_markdown_docs/assets/images/image83.png new file mode 100755 index 0000000000..fb54375a00 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image83.png differ diff --git a/docs/r_markdown_docs/assets/images/image85.png b/docs/r_markdown_docs/assets/images/image85.png new file mode 100755 index 0000000000..fcb0f9f4ad Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image85.png differ diff --git a/docs/r_markdown_docs/assets/images/image87.png b/docs/r_markdown_docs/assets/images/image87.png new file mode 100755 index 0000000000..36b4206c93 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image87.png differ diff --git a/docs/r_markdown_docs/assets/images/image88.png b/docs/r_markdown_docs/assets/images/image88.png new file mode 100755 index 0000000000..fd282e41d8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image88.png differ diff --git a/docs/r_markdown_docs/assets/images/image89.png b/docs/r_markdown_docs/assets/images/image89.png new file mode 100755 index 0000000000..067433f499 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image89.png differ diff --git a/docs/r_markdown_docs/assets/images/image90.png b/docs/r_markdown_docs/assets/images/image90.png new file mode 100755 index 0000000000..5b03b66b6a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image90.png differ diff --git a/docs/r_markdown_docs/assets/images/image92.png b/docs/r_markdown_docs/assets/images/image92.png new file mode 100755 index 0000000000..e7109d347f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image92.png differ diff --git a/docs/r_markdown_docs/assets/images/image93.png b/docs/r_markdown_docs/assets/images/image93.png new file mode 100755 index 0000000000..076e7f3a61 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image93.png differ diff --git a/docs/r_markdown_docs/assets/images/image94.png b/docs/r_markdown_docs/assets/images/image94.png new file mode 100755 index 0000000000..a90e411b41 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image94.png differ diff --git a/docs/r_markdown_docs/assets/images/image95.png b/docs/r_markdown_docs/assets/images/image95.png new file mode 100755 index 0000000000..aeaaeb7d85 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image95.png differ diff --git a/docs/r_markdown_docs/assets/images/image_upload_dialog.png b/docs/r_markdown_docs/assets/images/image_upload_dialog.png new file mode 100644 index 0000000000..1175d8e723 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/image_upload_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/k-means-cluster.png b/docs/r_markdown_docs/assets/images/k-means-cluster.png new file mode 100644 index 0000000000..402aed447d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/k-means-cluster.png differ diff --git a/docs/r_markdown_docs/assets/images/kinship-inbreeding.png b/docs/r_markdown_docs/assets/images/kinship-inbreeding.png new file mode 100644 index 0000000000..6e44141ab4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/kinship-inbreeding.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_add_items.png b/docs/r_markdown_docs/assets/images/list_manager_add_items.png new file mode 100644 index 0000000000..6e039b01a7 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_add_items.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_added_items.png b/docs/r_markdown_docs/assets/images/list_manager_added_items.png new file mode 100644 index 0000000000..8fadffec0b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_added_items.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_list_types.png b/docs/r_markdown_docs/assets/images/list_manager_list_types.png new file mode 100644 index 0000000000..74b475bd4a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_list_types.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_new_list.png b/docs/r_markdown_docs/assets/images/list_manager_new_list.png new file mode 100644 index 0000000000..fbd150b509 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_new_list.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_start.png b/docs/r_markdown_docs/assets/images/list_manager_start.png new file mode 100644 index 0000000000..85fa86f5fb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_start.png differ diff --git a/docs/r_markdown_docs/assets/images/list_manager_view_list.png b/docs/r_markdown_docs/assets/images/list_manager_view_list.png new file mode 100644 index 0000000000..e41ff230bd Binary files /dev/null and b/docs/r_markdown_docs/assets/images/list_manager_view_list.png differ diff --git a/docs/r_markdown_docs/assets/images/location_map.png b/docs/r_markdown_docs/assets/images/location_map.png new file mode 100644 index 0000000000..1f6283c606 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/location_map.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_cv.png b/docs/r_markdown_docs/assets/images/manage_NIRS_cv.png new file mode 100644 index 0000000000..25e92ce483 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_cv.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_export_model.png b/docs/r_markdown_docs/assets/images/manage_NIRS_export_model.png new file mode 100644 index 0000000000..7e2b044152 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_export_model.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_main_display.png b/docs/r_markdown_docs/assets/images/manage_NIRS_main_display.png new file mode 100644 index 0000000000..86e96e7973 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_main_display.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_results.png b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_results.png new file mode 100644 index 0000000000..8f2acd3632 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_results.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_dataset.png b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_dataset.png new file mode 100644 index 0000000000..91d8f9a5a3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_dataset.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_intro.png b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_intro.png new file mode 100644 index 0000000000..d6e9d1d701 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_prediction_workflow_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_select_model.png b/docs/r_markdown_docs/assets/images/manage_NIRS_select_model.png new file mode 100644 index 0000000000..a6a52bed9c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_select_model.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_NIRS_snv.png b/docs/r_markdown_docs/assets/images/manage_NIRS_snv.png new file mode 100644 index 0000000000..ad8b6d8240 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_NIRS_snv.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_file.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_file.png new file mode 100644 index 0000000000..d05d1c5d5a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_file.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_list.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_list.png new file mode 100644 index 0000000000..02de06670d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_complete_using_list.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_found.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_found.png new file mode 100644 index 0000000000..247ec13606 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_found.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy.png new file mode 100644 index 0000000000..314d1a994b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy_options.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy_options.png new file mode 100644 index 0000000000..44cbbb4f9c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_fuzzy_options.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_link.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_link.png new file mode 100644 index 0000000000..7a8b0119e9 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_link.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_saved.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_saved.png new file mode 100644 index 0000000000..aaa65bc317 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_saved.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_spreadsheet_info.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_spreadsheet_info.png new file mode 100644 index 0000000000..e87c20ab51 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_spreadsheet_info.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_file.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_file.png new file mode 100644 index 0000000000..17974c6709 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_file.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_lists.png b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_lists.png new file mode 100644 index 0000000000..b6d0933106 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_accessions_add_accessions_using_lists.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_cross_click_cross_wishlist.png b/docs/r_markdown_docs/assets/images/manage_cross_click_cross_wishlist.png new file mode 100644 index 0000000000..53520e7fa6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_cross_click_cross_wishlist.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_crosses1.png b/docs/r_markdown_docs/assets/images/manage_crosses1.png new file mode 100644 index 0000000000..0399c65e82 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_crosses1.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_crosses2.png b/docs/r_markdown_docs/assets/images/manage_crosses2.png new file mode 100644 index 0000000000..99c4efd6a9 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_crosses2.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_crosses_addcross.png b/docs/r_markdown_docs/assets/images/manage_crosses_addcross.png new file mode 100644 index 0000000000..eef2532f9c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_crosses_addcross.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_crosses_crossingtrial.png b/docs/r_markdown_docs/assets/images/manage_crosses_crossingtrial.png new file mode 100644 index 0000000000..4163afa0a4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_crosses_crossingtrial.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_crosses_upload.png b/docs/r_markdown_docs/assets/images/manage_crosses_upload.png new file mode 100644 index 0000000000..ece8c15cb8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_crosses_upload.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_button.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_button.png new file mode 100644 index 0000000000..292054a2ac Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_button.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_intro.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_intro.png new file mode 100644 index 0000000000..4881cafdd2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_project.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_project.png new file mode 100644 index 0000000000..663c42af78 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_project.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_protocol.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_protocol.png new file mode 100644 index 0000000000..9f98f3601e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_protocol.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_vcf.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_vcf.png new file mode 100644 index 0000000000..3816349b63 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_upload_dialog_vcf.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_download_prot.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_download_prot.png new file mode 100644 index 0000000000..6f5fadfb31 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_download_prot.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_trial.png b/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_trial.png new file mode 100644 index 0000000000..3cb2661ef1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_data_wizard_trial.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials.png new file mode 100644 index 0000000000..fa4db1c5d0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial.png new file mode 100644 index 0000000000..97914dca99 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_confirm.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_confirm.png new file mode 100644 index 0000000000..8e77c1fc1b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_confirm.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_wells.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_wells.png new file mode 100644 index 0000000000..f7b15f8b84 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_add_trial_wells.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_info.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_info.png new file mode 100644 index 0000000000..a0caac3049 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_info.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_table.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_table.png new file mode 100644 index 0000000000..b5bfee2e53 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_detail_table.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_expand.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_expand.png new file mode 100644 index 0000000000..19d0d2143b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_expand.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_genotyping_trials_upload_template.png b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_upload_template.png new file mode 100644 index 0000000000..7b52c36662 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_genotyping_trials_upload_template.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_apply.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_apply.png new file mode 100644 index 0000000000..eda704ba07 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_apply.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_button.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_button.png new file mode 100644 index 0000000000..b5598ccff2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_button.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_cropping.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_cropping.png new file mode 100644 index 0000000000..dc6fed6107 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_cropping.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_drone_run_band.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_drone_run_band.png new file mode 100644 index 0000000000..1e90819cc5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_drone_run_band.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_intro.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_intro.png new file mode 100644 index 0000000000..066e087c18 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_phenotypes.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_phenotypes.png new file mode 100644 index 0000000000..c91faad77d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_phenotypes.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_plot_polygon.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_plot_polygon.png new file mode 100644 index 0000000000..124f980289 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_plot_polygon.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_rotate.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_rotate.png new file mode 100644 index 0000000000..0d6c81305f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_rotate.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_threshold_hist.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_threshold_hist.png new file mode 100644 index 0000000000..76ee2c1a1d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_threshold_hist.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_vegetation_index.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_vegetation_index.png new file mode 100644 index 0000000000..779728165e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_standard_process_vegetation_index.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_drone_run.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_drone_run.png new file mode 100644 index 0000000000..155527b1be Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_drone_run.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_field_trial.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_field_trial.png new file mode 100644 index 0000000000..df5781b2a1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_field_trial.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_1.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_1.png new file mode 100644 index 0000000000..126dea631b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_1.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_2.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_2.png new file mode 100644 index 0000000000..655db9603c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_2.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_3.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_3.png new file mode 100644 index 0000000000..e24d606312 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_image_info_3.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_rasters.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_rasters.png new file mode 100644 index 0000000000..dc09d0d0d1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_rasters.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_zipfile.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_zipfile.png new file mode 100644 index 0000000000..a3ca997d82 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_images_zipfile.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_intro.png b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_intro.png new file mode 100644 index 0000000000..ee8d1ed394 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_image_phenotyping_upload_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_start_button.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_start_button.png new file mode 100644 index 0000000000..68d63ea624 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_start_button.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_intro.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_intro.png new file mode 100644 index 0000000000..fc5df2770f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_method.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_method.png new file mode 100644 index 0000000000..b596abddb1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_method.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_observation_variable.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_observation_variable.png new file mode 100644 index 0000000000..bef57ebf6e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_observation_variable.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_scale.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_scale.png new file mode 100644 index 0000000000..b123feffd8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_scale.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_submit.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_submit.png new file mode 100644 index 0000000000..1115683746 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_submit.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_trait.png b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_trait.png new file mode 100644 index 0000000000..52f93ce544 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_observation_variables_workflow_trait.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_populations.png b/docs/r_markdown_docs/assets/images/manage_populations.png new file mode 100644 index 0000000000..f4993eb4c5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_populations.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_populations_add_population.png b/docs/r_markdown_docs/assets/images/manage_populations_add_population.png new file mode 100644 index 0000000000..5c6bcd1fa6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_populations_add_population.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_populations_expand_table.png b/docs/r_markdown_docs/assets/images/manage_populations_expand_table.png new file mode 100644 index 0000000000..e292627c59 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_populations_expand_table.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_populations_table.png b/docs/r_markdown_docs/assets/images/manage_populations_table.png new file mode 100644 index 0000000000..064260ec0e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_populations_table.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_accession_detail.png b/docs/r_markdown_docs/assets/images/manage_seedlot_accession_detail.png new file mode 100644 index 0000000000..f399ac437f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_accession_detail.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_accession_list_search.png b/docs/r_markdown_docs/assets/images/manage_seedlot_accession_list_search.png new file mode 100644 index 0000000000..b841ddbeae Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_accession_list_search.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_add_seedlot.png b/docs/r_markdown_docs/assets/images/manage_seedlot_add_seedlot.png new file mode 100644 index 0000000000..f5c6a7b722 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_add_seedlot.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_list_available_seedlots.png b/docs/r_markdown_docs/assets/images/manage_seedlot_list_available_seedlots.png new file mode 100644 index 0000000000..83f289041e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_list_available_seedlots.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory.png b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory.png new file mode 100644 index 0000000000..601ad72fe8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory_template.png b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory_template.png new file mode 100644 index 0000000000..1188565c28 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_inventory_template.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlot_upload_seedlots.png b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_seedlots.png new file mode 100644 index 0000000000..c8345df07e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlot_upload_seedlots.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlots.png b/docs/r_markdown_docs/assets/images/manage_seedlots.png new file mode 100644 index 0000000000..6071a006ee Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlots.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_seedlots_seedlot_maintenance.png b/docs/r_markdown_docs/assets/images/manage_seedlots_seedlot_maintenance.png new file mode 100644 index 0000000000..0c849fa69b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_seedlots_seedlot_maintenance.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_button.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_button.png new file mode 100644 index 0000000000..0e7641195a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_button.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_download.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_download.png new file mode 100644 index 0000000000..7eac3b7c4d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_download.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_intro.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_intro.png new file mode 100644 index 0000000000..5d6e500961 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_intro.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants.png new file mode 100644 index 0000000000..481ef48f44 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissues.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissues.png new file mode 100644 index 0000000000..069cdddb09 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissues.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_select.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_select.png new file mode 100644 index 0000000000..5420ad95ec Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_select.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_submit.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_submit.png new file mode 100644 index 0000000000..e062f6060f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_field_trial_submit.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_button.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_button.png new file mode 100644 index 0000000000..c3574146dc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_button.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_download.png b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_download.png new file mode 100644 index 0000000000..16b3d5a274 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_tissue_samples_create_tissues_genotyping_trial_download.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials.png b/docs/r_markdown_docs/assets/images/manage_trials.png new file mode 100644 index 0000000000..1388570f9c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_create.png b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_create.png new file mode 100644 index 0000000000..f60504aed4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_create.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_default.png b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_default.png new file mode 100644 index 0000000000..17e445789e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_default.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_detail.png b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_detail.png new file mode 100644 index 0000000000..bb187f48bb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_detail.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_related_stock.png b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_related_stock.png new file mode 100644 index 0000000000..8315708f36 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_tissue_sample_related_stock.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_tissue_samples.png b/docs/r_markdown_docs/assets/images/manage_trials_tissue_samples.png new file mode 100644 index 0000000000..56255706c9 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_tissue_samples.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_1.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_1.png new file mode 100644 index 0000000000..98636bec52 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_1.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_2.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_2.png new file mode 100644 index 0000000000..06e95f323d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_2.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_3.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_3.png new file mode 100644 index 0000000000..a9d23c486d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_3.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_4.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_4.png new file mode 100644 index 0000000000..d5ac03dec0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_4.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_5.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_5.png new file mode 100644 index 0000000000..ef4d8bec05 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_5.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_6.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_6.png new file mode 100644 index 0000000000..0766081d47 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_6.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_7.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_7.png new file mode 100644 index 0000000000..82f077e640 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_7.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_8.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_8.png new file mode 100644 index 0000000000..8f9a36dfc0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_8.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_template.png b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_template.png new file mode 100644 index 0000000000..70b95f6316 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_trials_upload_trial_template.png differ diff --git a/docs/r_markdown_docs/assets/images/manage_user_roles_page.png b/docs/r_markdown_docs/assets/images/manage_user_roles_page.png new file mode 100644 index 0000000000..6372d1f0f3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/manage_user_roles_page.png differ diff --git a/docs/r_markdown_docs/assets/images/mixedmodel_tool_model_build_step.png b/docs/r_markdown_docs/assets/images/mixedmodel_tool_model_build_step.png new file mode 100644 index 0000000000..d97a24f3b1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/mixedmodel_tool_model_build_step.png differ diff --git a/docs/r_markdown_docs/assets/images/multiple_trial_upload_with_email.png b/docs/r_markdown_docs/assets/images/multiple_trial_upload_with_email.png new file mode 100644 index 0000000000..df9571559a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/multiple_trial_upload_with_email.png differ diff --git a/docs/r_markdown_docs/assets/images/new_cross_dialog2.png b/docs/r_markdown_docs/assets/images/new_cross_dialog2.png new file mode 100644 index 0000000000..619abfe947 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/new_cross_dialog2.png differ diff --git a/docs/r_markdown_docs/assets/images/new_cross_dialog_type.png b/docs/r_markdown_docs/assets/images/new_cross_dialog_type.png new file mode 100644 index 0000000000..6e346fd048 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/new_cross_dialog_type.png differ diff --git a/docs/r_markdown_docs/assets/images/new_program_screenshot.png b/docs/r_markdown_docs/assets/images/new_program_screenshot.png new file mode 100644 index 0000000000..812582a64a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/new_program_screenshot.png differ diff --git a/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_graph.png b/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_graph.png new file mode 100644 index 0000000000..f16a61381e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_graph.png differ diff --git a/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_tree.png b/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_tree.png new file mode 100644 index 0000000000..e8f38267fa Binary files /dev/null and b/docs/r_markdown_docs/assets/images/odk_ona_cross_progress_tree.png differ diff --git a/docs/r_markdown_docs/assets/images/odk_ona_cross_status.png b/docs/r_markdown_docs/assets/images/odk_ona_cross_status.png new file mode 100644 index 0000000000..d3038d9631 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/odk_ona_cross_status.png differ diff --git a/docs/r_markdown_docs/assets/images/odk_ona_management.png b/docs/r_markdown_docs/assets/images/odk_ona_management.png new file mode 100644 index 0000000000..988c5faabf Binary files /dev/null and b/docs/r_markdown_docs/assets/images/odk_ona_management.png differ diff --git a/docs/r_markdown_docs/assets/images/ontology_browser.png b/docs/r_markdown_docs/assets/images/ontology_browser.png new file mode 100644 index 0000000000..a28e7b66d4 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ontology_browser.png differ diff --git a/docs/r_markdown_docs/assets/images/ontology_browser_variable.png b/docs/r_markdown_docs/assets/images/ontology_browser_variable.png new file mode 100644 index 0000000000..2ae6369801 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/ontology_browser_variable.png differ diff --git a/docs/r_markdown_docs/assets/images/outliers_dataset_actions.png b/docs/r_markdown_docs/assets/images/outliers_dataset_actions.png new file mode 100644 index 0000000000..19c00444d2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/outliers_dataset_actions.png differ diff --git a/docs/r_markdown_docs/assets/images/outliers_dataset_basic_panel.png b/docs/r_markdown_docs/assets/images/outliers_dataset_basic_panel.png new file mode 100644 index 0000000000..db3d362172 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/outliers_dataset_basic_panel.png differ diff --git a/docs/r_markdown_docs/assets/images/outliers_dataset_full_visualisation.png b/docs/r_markdown_docs/assets/images/outliers_dataset_full_visualisation.png new file mode 100644 index 0000000000..cf3e27b67d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/outliers_dataset_full_visualisation.png differ diff --git a/docs/r_markdown_docs/assets/images/outliers_dataset_icon.png b/docs/r_markdown_docs/assets/images/outliers_dataset_icon.png new file mode 100644 index 0000000000..46c906ab82 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/outliers_dataset_icon.png differ diff --git a/docs/r_markdown_docs/assets/images/outliers_dataset_panel_with_set.png b/docs/r_markdown_docs/assets/images/outliers_dataset_panel_with_set.png new file mode 100644 index 0000000000..c9ea942e10 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/outliers_dataset_panel_with_set.png differ diff --git a/docs/r_markdown_docs/assets/images/pca_iita_naccri_trials.png b/docs/r_markdown_docs/assets/images/pca_iita_naccri_trials.png new file mode 100644 index 0000000000..d299d99d24 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/pca_iita_naccri_trials.png differ diff --git a/docs/r_markdown_docs/assets/images/pedigree_upload_format.png b/docs/r_markdown_docs/assets/images/pedigree_upload_format.png new file mode 100644 index 0000000000..624aed2591 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/pedigree_upload_format.png differ diff --git a/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_dialog.png b/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_dialog.png new file mode 100644 index 0000000000..bd9804071f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_help_dialog.png b/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_help_dialog.png new file mode 100644 index 0000000000..7da6bd84c2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/phenotype_spreadsheet_upload_help_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/plot_pheno_barcode.png b/docs/r_markdown_docs/assets/images/plot_pheno_barcode.png new file mode 100644 index 0000000000..14d71869c7 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/plot_pheno_barcode.png differ diff --git a/docs/r_markdown_docs/assets/images/progenies_upload_spreadsheet.png b/docs/r_markdown_docs/assets/images/progenies_upload_spreadsheet.png new file mode 100644 index 0000000000..f44f5ef71a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/progenies_upload_spreadsheet.png differ diff --git a/docs/r_markdown_docs/assets/images/replace_plot_accession_form.png b/docs/r_markdown_docs/assets/images/replace_plot_accession_form.png new file mode 100644 index 0000000000..970cff2460 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/replace_plot_accession_form.png differ diff --git a/docs/r_markdown_docs/assets/images/roles.png b/docs/r_markdown_docs/assets/images/roles.png new file mode 100644 index 0000000000..1b98be0580 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/roles.png differ diff --git a/docs/r_markdown_docs/assets/images/search_accessions.png b/docs/r_markdown_docs/assets/images/search_accessions.png new file mode 100644 index 0000000000..168dfd2711 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_accessions.png differ diff --git a/docs/r_markdown_docs/assets/images/search_accessions_graphical_filtering.png b/docs/r_markdown_docs/assets/images/search_accessions_graphical_filtering.png new file mode 100644 index 0000000000..a75823cf06 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_accessions_graphical_filtering.png differ diff --git a/docs/r_markdown_docs/assets/images/search_accessions_properties_search.png b/docs/r_markdown_docs/assets/images/search_accessions_properties_search.png new file mode 100644 index 0000000000..7275f53b13 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_accessions_properties_search.png differ diff --git a/docs/r_markdown_docs/assets/images/search_accessions_properties_view.png b/docs/r_markdown_docs/assets/images/search_accessions_properties_view.png new file mode 100644 index 0000000000..606d22be85 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_accessions_properties_view.png differ diff --git a/docs/r_markdown_docs/assets/images/search_seedlots.png b/docs/r_markdown_docs/assets/images/search_seedlots.png new file mode 100644 index 0000000000..e0b79c1823 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_seedlots.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_grm.png b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_grm.png new file mode 100644 index 0000000000..def7d18451 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_grm.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_gwas.png b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_gwas.png new file mode 100644 index 0000000000..6bad710e1a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_gwas.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_manhattan_plot.png b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_manhattan_plot.png new file mode 100644 index 0000000000..cb7e19c1bd Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_manhattan_plot.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_qq_plot.png b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_qq_plot.png new file mode 100644 index 0000000000..0584d38971 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotype_analyses_qq_plot.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_genotypes.png b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_genotypes.png new file mode 100644 index 0000000000..4f3a5a5f58 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_genotypes.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_main.png b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_main.png new file mode 100644 index 0000000000..71deddbe73 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_main.png differ diff --git a/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_phenotypes.png b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_phenotypes.png new file mode 100644 index 0000000000..1baa824e12 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/search_wizard_genotyping_protocols_download_phenotypes.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_add_new_transaction.png b/docs/r_markdown_docs/assets/images/seedlot_add_new_transaction.png new file mode 100644 index 0000000000..87ed2a92c1 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_add_new_transaction.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_detail_page.png b/docs/r_markdown_docs/assets/images/seedlot_detail_page.png new file mode 100644 index 0000000000..856e9d5dba Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_detail_page.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_filtered.png b/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_filtered.png new file mode 100644 index 0000000000..afc2005c2b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_filtered.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_unfiltered.png b/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_unfiltered.png new file mode 100644 index 0000000000..d4d18ba25f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_maintenance_events_unfiltered.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_maintenance_record.png b/docs/r_markdown_docs/assets/images/seedlot_maintenance_record.png new file mode 100644 index 0000000000..135f0b8a80 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_maintenance_record.png differ diff --git a/docs/r_markdown_docs/assets/images/seedlot_maintenance_upload.png b/docs/r_markdown_docs/assets/images/seedlot_maintenance_upload.png new file mode 100644 index 0000000000..5c31f08875 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/seedlot_maintenance_upload.png differ diff --git a/docs/r_markdown_docs/assets/images/selection_proportion_clustering.png b/docs/r_markdown_docs/assets/images/selection_proportion_clustering.png new file mode 100644 index 0000000000..f8f331bcdf Binary files /dev/null and b/docs/r_markdown_docs/assets/images/selection_proportion_clustering.png differ diff --git a/docs/r_markdown_docs/assets/images/sequence_metadata_manage.png b/docs/r_markdown_docs/assets/images/sequence_metadata_manage.png new file mode 100644 index 0000000000..c6522103dc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/sequence_metadata_manage.png differ diff --git a/docs/r_markdown_docs/assets/images/sequence_metadata_search_advanced.png b/docs/r_markdown_docs/assets/images/sequence_metadata_search_advanced.png new file mode 100644 index 0000000000..ec052e4e6c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/sequence_metadata_search_advanced.png differ diff --git a/docs/r_markdown_docs/assets/images/sequence_metadata_search_basic.png b/docs/r_markdown_docs/assets/images/sequence_metadata_search_basic.png new file mode 100644 index 0000000000..30318e26af Binary files /dev/null and b/docs/r_markdown_docs/assets/images/sequence_metadata_search_basic.png differ diff --git a/docs/r_markdown_docs/assets/images/sequence_metadata_search_results.png b/docs/r_markdown_docs/assets/images/sequence_metadata_search_results.png new file mode 100644 index 0000000000..ededb22b20 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/sequence_metadata_search_results.png differ diff --git a/docs/r_markdown_docs/assets/images/spatial_layout_usage_help.png b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help.png new file mode 100644 index 0000000000..141765b324 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help.png differ diff --git a/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_1.png b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_1.png new file mode 100644 index 0000000000..5d68f69978 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_1.png differ diff --git a/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_2.png b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_2.png new file mode 100644 index 0000000000..3fce0919f0 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_2.png differ diff --git a/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_3.png b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_3.png new file mode 100644 index 0000000000..0aa1b38d48 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/spatial_layout_usage_help_3.png differ diff --git a/docs/r_markdown_docs/assets/images/stock_add_image.png b/docs/r_markdown_docs/assets/images/stock_add_image.png new file mode 100644 index 0000000000..feb9dfa254 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/stock_add_image.png differ diff --git a/docs/r_markdown_docs/assets/images/tool_compatibility_data_summary.png b/docs/r_markdown_docs/assets/images/tool_compatibility_data_summary.png new file mode 100644 index 0000000000..c0a2d97ed2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/tool_compatibility_data_summary.png differ diff --git a/docs/r_markdown_docs/assets/images/tool_compatibility_details.png b/docs/r_markdown_docs/assets/images/tool_compatibility_details.png new file mode 100644 index 0000000000..e4876d8a9c Binary files /dev/null and b/docs/r_markdown_docs/assets/images/tool_compatibility_details.png differ diff --git a/docs/r_markdown_docs/assets/images/trait-search-default.png b/docs/r_markdown_docs/assets/images/trait-search-default.png new file mode 100644 index 0000000000..e81814011b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trait-search-default.png differ diff --git a/docs/r_markdown_docs/assets/images/trait-search.png b/docs/r_markdown_docs/assets/images/trait-search.png new file mode 100644 index 0000000000..bc7da3f71d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trait-search.png differ diff --git a/docs/r_markdown_docs/assets/images/transcript_details_example.png b/docs/r_markdown_docs/assets/images/transcript_details_example.png new file mode 100644 index 0000000000..2ebe7b9024 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/transcript_details_example.png differ diff --git a/docs/r_markdown_docs/assets/images/transcript_details_headers.png b/docs/r_markdown_docs/assets/images/transcript_details_headers.png new file mode 100644 index 0000000000..0391301b8b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/transcript_details_headers.png differ diff --git a/docs/r_markdown_docs/assets/images/transcriptomics_menu.png b/docs/r_markdown_docs/assets/images/transcriptomics_menu.png new file mode 100644 index 0000000000..314b8094e2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/transcriptomics_menu.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_barcode.png b/docs/r_markdown_docs/assets/images/trial_barcode.png new file mode 100644 index 0000000000..209493e6e9 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_barcode.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_1.png b/docs/r_markdown_docs/assets/images/trial_create_form_1.png new file mode 100644 index 0000000000..16a465448b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_1.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_2.png b/docs/r_markdown_docs/assets/images/trial_create_form_2.png new file mode 100644 index 0000000000..6c0e80b973 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_2.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_3.png b/docs/r_markdown_docs/assets/images/trial_create_form_3.png new file mode 100644 index 0000000000..716f37e0c2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_3.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_4.png b/docs/r_markdown_docs/assets/images/trial_create_form_4.png new file mode 100644 index 0000000000..7057b9ea7a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_4.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_5.png b/docs/r_markdown_docs/assets/images/trial_create_form_5.png new file mode 100644 index 0000000000..c544c04144 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_5.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_6.png b/docs/r_markdown_docs/assets/images/trial_create_form_6.png new file mode 100644 index 0000000000..80b405e5ba Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_6.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_7_bottom.png b/docs/r_markdown_docs/assets/images/trial_create_form_7_bottom.png new file mode 100644 index 0000000000..ce10a0f747 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_7_bottom.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_7_middle.png b/docs/r_markdown_docs/assets/images/trial_create_form_7_middle.png new file mode 100644 index 0000000000..34ba171db2 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_7_middle.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_7_top.png b/docs/r_markdown_docs/assets/images/trial_create_form_7_top.png new file mode 100644 index 0000000000..98ad9300ff Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_7_top.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_form_8.png b/docs/r_markdown_docs/assets/images/trial_create_form_8.png new file mode 100644 index 0000000000..6583861ded Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_form_8.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_manage_trials.png b/docs/r_markdown_docs/assets/images/trial_create_manage_trials.png new file mode 100644 index 0000000000..a02b812ae8 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_manage_trials.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_create_open_form.png b/docs/r_markdown_docs/assets/images/trial_create_open_form.png new file mode 100644 index 0000000000..e9782cb87a Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_create_open_form.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries.png new file mode 100644 index 0000000000..022eb51c1d Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png new file mode 100644 index 0000000000..96a5ac7d75 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_autogenerated.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload.png new file mode 100644 index 0000000000..d95e484ffc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload_info.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload_info.png new file mode 100644 index 0000000000..33f8110138 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plant_entries_upload_info.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps.png new file mode 100644 index 0000000000..b46775ca34 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_dialog.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_dialog.png new file mode 100644 index 0000000000..ef962f033e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_dialog.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_upload_info.png b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_upload_info.png new file mode 100644 index 0000000000..72e5f9b79e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_add_plot_gps_upload_info.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait1.png b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait1.png new file mode 100644 index 0000000000..b39238e058 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait1.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait2.png b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait2.png new file mode 100644 index 0000000000..649491ccd6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait2.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait3.png b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait3.png new file mode 100644 index 0000000000..c859a8477b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_delete_trait3.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_navigator.png b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator.png new file mode 100644 index 0000000000..c8d856bb5f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_1.png b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_1.png new file mode 100644 index 0000000000..0c86fb6b52 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_1.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_2.png b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_2.png new file mode 100644 index 0000000000..43f1f7b5bf Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_2.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_3.png b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_3.png new file mode 100644 index 0000000000..f70afc2eea Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_navigator_3.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_start.png b/docs/r_markdown_docs/assets/images/trial_detail_page_start.png new file mode 100644 index 0000000000..71ae4ddc57 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_start.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_suppress_phenotype.png b/docs/r_markdown_docs/assets/images/trial_detail_page_suppress_phenotype.png new file mode 100644 index 0000000000..c7604288ca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_suppress_phenotype.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_trait_heatmap.png b/docs/r_markdown_docs/assets/images/trial_detail_page_trait_heatmap.png new file mode 100644 index 0000000000..baee7c1529 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_trait_heatmap.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_detail_page_view_repetitive_measurements.png b/docs/r_markdown_docs/assets/images/trial_detail_page_view_repetitive_measurements.png new file mode 100644 index 0000000000..903be8d86b Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_detail_page_view_repetitive_measurements.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_meta_data_download.png b/docs/r_markdown_docs/assets/images/trial_meta_data_download.png new file mode 100644 index 0000000000..d9abb89ebf Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_meta_data_download.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_search.png b/docs/r_markdown_docs/assets/images/trial_search.png new file mode 100644 index 0000000000..5ce8d8f394 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_search.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_upload_additional_file.png b/docs/r_markdown_docs/assets/images/trial_upload_additional_file.png new file mode 100644 index 0000000000..ac379ca729 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_upload_additional_file.png differ diff --git a/docs/r_markdown_docs/assets/images/trial_upload_additional_file_info.png b/docs/r_markdown_docs/assets/images/trial_upload_additional_file_info.png new file mode 100644 index 0000000000..15e3cc48fb Binary files /dev/null and b/docs/r_markdown_docs/assets/images/trial_upload_additional_file_info.png differ diff --git a/docs/r_markdown_docs/assets/images/update_trial_metadata.png b/docs/r_markdown_docs/assets/images/update_trial_metadata.png new file mode 100644 index 0000000000..eb04c25b21 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/update_trial_metadata.png differ diff --git a/docs/r_markdown_docs/assets/images/upload_NIRS_example_format.png b/docs/r_markdown_docs/assets/images/upload_NIRS_example_format.png new file mode 100644 index 0000000000..282783a3ee Binary files /dev/null and b/docs/r_markdown_docs/assets/images/upload_NIRS_example_format.png differ diff --git a/docs/r_markdown_docs/assets/images/upload_NIRS_plot_example.png b/docs/r_markdown_docs/assets/images/upload_NIRS_plot_example.png new file mode 100644 index 0000000000..882f90c2ca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/upload_NIRS_plot_example.png differ diff --git a/docs/r_markdown_docs/assets/images/upload_crosses.png b/docs/r_markdown_docs/assets/images/upload_crosses.png new file mode 100644 index 0000000000..3554bc3026 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/upload_crosses.png differ diff --git a/docs/r_markdown_docs/assets/images/vectorButtons.png b/docs/r_markdown_docs/assets/images/vectorButtons.png new file mode 100644 index 0000000000..e047305a31 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/vectorButtons.png differ diff --git a/docs/r_markdown_docs/assets/images/vectorHighlight.png b/docs/r_markdown_docs/assets/images/vectorHighlight.png new file mode 100644 index 0000000000..dc858c617e Binary files /dev/null and b/docs/r_markdown_docs/assets/images/vectorHighlight.png differ diff --git a/docs/r_markdown_docs/assets/images/vectorViewer.png b/docs/r_markdown_docs/assets/images/vectorViewer.png new file mode 100644 index 0000000000..6e47081717 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/vectorViewer.png differ diff --git a/docs/r_markdown_docs/assets/images/waves_breedbase_schema.png b/docs/r_markdown_docs/assets/images/waves_breedbase_schema.png new file mode 100644 index 0000000000..35c79f61e6 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/waves_breedbase_schema.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle.jpg b/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle.jpg new file mode 100644 index 0000000000..b93f2baafe Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle.jpg differ diff --git a/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle_min_details.jpg b/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle_min_details.jpg new file mode 100644 index 0000000000..5445ccf87f Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_any_min_all_toggle_min_details.jpg differ diff --git a/docs/r_markdown_docs/assets/images/wizard_download_options.png b/docs/r_markdown_docs/assets/images/wizard_download_options.png new file mode 100644 index 0000000000..89b9ff12e3 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_download_options.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_interface.png b/docs/r_markdown_docs/assets/images/wizard_interface.png new file mode 100644 index 0000000000..5d21c76547 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_interface.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_interface_selections.png b/docs/r_markdown_docs/assets/images/wizard_interface_selections.png new file mode 100644 index 0000000000..89441ce986 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_interface_selections.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_load_create_dataset.png b/docs/r_markdown_docs/assets/images/wizard_load_create_dataset.png new file mode 100644 index 0000000000..156ac0d2dc Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_load_create_dataset.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_metadata_download.png b/docs/r_markdown_docs/assets/images/wizard_metadata_download.png new file mode 100644 index 0000000000..e7796e0120 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_metadata_download.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_related_metadata_download.png b/docs/r_markdown_docs/assets/images/wizard_related_metadata_download.png new file mode 100644 index 0000000000..f286748635 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_related_metadata_download.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_related_phenotypes_download.png b/docs/r_markdown_docs/assets/images/wizard_related_phenotypes_download.png new file mode 100644 index 0000000000..c2ca9c3314 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_related_phenotypes_download.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_save_current_selection.png b/docs/r_markdown_docs/assets/images/wizard_save_current_selection.png new file mode 100644 index 0000000000..85639582d5 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_save_current_selection.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_select_list.png b/docs/r_markdown_docs/assets/images/wizard_select_list.png new file mode 100644 index 0000000000..368ed0b7ca Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_select_list.png differ diff --git a/docs/r_markdown_docs/assets/images/wizard_update_list_refresh.png b/docs/r_markdown_docs/assets/images/wizard_update_list_refresh.png new file mode 100644 index 0000000000..3ce5b3dd84 Binary files /dev/null and b/docs/r_markdown_docs/assets/images/wizard_update_list_refresh.png differ diff --git a/docs/r_markdown_docs/assets/style.css b/docs/r_markdown_docs/assets/style.css new file mode 100644 index 0000000000..046f3253f8 --- /dev/null +++ b/docs/r_markdown_docs/assets/style.css @@ -0,0 +1,15 @@ +/*--- LOGO ---*/ + +.toc-logo { + width: 200px !important; + object-fit: contain; + margin: 0 auto; +} + +.toc-logo img { + max-width: 100%; +} + +.tab { + margin-left: 40px; +} \ No newline at end of file diff --git a/docs/r_markdown_docs/basic_website_usage.Rmd b/docs/r_markdown_docs/basic_website_usage.Rmd new file mode 100644 index 0000000000..afd3d5bf35 --- /dev/null +++ b/docs/r_markdown_docs/basic_website_usage.Rmd @@ -0,0 +1,236 @@ +--- +output: + word_document: default + html_document: default + pdf_document: default +--- + +# Basic Website Usage + +Breedbase is usually hosted on the cloud and is entirely web-based, with only a browser required to access and use it. The recommended browser is Firefox. To use Breedbase, you or your project will usually need to have your own Breedbase instance. Information how you can obtain your own instance can be obtained from the Breedbase project (https://breedbase.org/). + +Once an instance is set up, the site needs to be configured and some metadata uploaded before you can design, run and analyze trials: + +- define user accounts and their access privileges. + +- add a trait ontology with the traits that you require. + +- add the locations that you use in your breeding program + +- add the foundational germplasm and the respective pedigree data + +- "historical" trial data can be uploaded as needed. Usually more recent trials are prioritized over older trials + +In this chapter, we will cover how you can manage the user accounts and some basic website features such as lists and how to navigate the menus. The other topics are covered in subsequent chapters. Note that Breedbase instances are highly customizable, and that not every instance of Breedbase will have the same options in the same location. Refer to the site specific documentation, if any, for site specific information. + +## Creating a User Account + +### Verifying first that you do not already have an account + +Before creating an account, please verify first that you don’t already have an account. You can use “Search” menu to check if you already registered as a user. + +In the "Search" menu, selecting the "People" tab and search your name. If nothing is found, proceed with the instructions below. Otherwise, continue by clicking the “Login” button. If you have forgotten your password, you can retrieve it by clicking the "Forgot your password?" link on the login page. + +### Creating a user account + +On the right of the toolbar, click on “Login", which will take you to the login dialog. On the login dialog, click on the link “sign up for an account.” It will take you to the page below: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image331.png') +``` + +Filling in all of the information, then click "Create Account." + +After you submit the information, an email will be sent to the provided email address. Check your email and click on the link to activate your account. + +## Managing your Account + +### Login + +To login, click the "Login" link in the toolbar on any page and enter your username and password. + +If you have forgotten your password, you can retrieve it by clicking the "Forgot your password?" link on the login page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image166.png') +``` + +### Editing Account Settings + +Account settings can be edited by clicking on the "my profile" link displayed as your user name, on the right of the toolbar. You must be logged in order to access and change account settings. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image261.png') +``` + +You can add personal information to your account using the "View or update personal information" link. + +To change your password, username, or your contact email, click on “Update account information” link. You must provide your old password before you can make any changes. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image144.png') +``` + +### Changing Your Account Status: From “User” to “Submitter” + +After you create an account, your account has a "user" status. This account has limited privileges. + +Accounts with “user” status are able to: + +- Change personal information +- Post comments on pages +- Post to the forum + +To upgrade your account status to "submitter,” contact the database curators using the "contact" link provided at the footer of each page. Submitter accounts can add data, such as new plots, accessions, phenotypic data and images. + +### Submitting Feedback on an SGN Database + +We appreciate your feedback! Feel free to submit any questions or suggestions by using the "Feedback" link provided at the footer of each page. + +## Menu Layout + +SGN Database websites have a toolbar on the top of each page with a number of menus for convenient access of major functions. The menus, as pictured below, are “search,” “manage,” “analyze,” and “maps.” The toolbar also provides a quick search, a “log in” button, and a “new user” button. The menus can be customized for each instance, so they may not appear exactly as shown on your instance; for example, most instances do not have a map menu. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image270.png') +``` + +### Menu Options + +#### Search {.unnumbered} + +In the Search menu, the options are: + +| Tab | Description | +|---------------|---------------------------------------------------------| +| Wizard | Search different accessions and plots by location, year, trial, and trait data. Can also be used to create lists of different types. | +| Accession and plots | Search accessions and plots using a variety of criteria | +| Trials | Search trials by name, description, breeding program, year, location, and trial type. | +| Markers | Search different markers | +| Images | Search images contained in the SGN database | +| People | Search database users | + +#### Manage {.unnumbered} + +In the Manage menu, the options are: + +| Tab | Description | +|----------------|--------------------------------------------------------| +| Breeding Programs | View, add and delete breeding programs | +| Locations | View, add and delete locations | +| Accessions | Manage and search different accessions | +| Seedlots | Manage and search different seedlots | +| Crosses | Create new crosses in the database | +| Field Trials | Manage field trials. Create trials using different field layouts. | +| Genotyping Plates | Manage genotyping plates. Create 96 or 384 well plates. | +| Phenotyping | Upload phenotyping files from the Tablet Field Book application | +| Field Book App | Manage the field book app data (download files to tablet) | +| Barcodes | Refers to the old barcode system, mainly historical | +| Download | Download information in the database based on lists | + +#### Analyze {.unnumbered} + +**Clicking on the "Analyze" link will give a full menu of all analysis functions**\ +In the Analyze menu, the options are: + +| Tab | Description | +|-----------------|-------------------------------------------------------| +| **Breeder Tools** | | +| Breeder Home | Access breeding functionalities. Lists important and helpful links. | +| Barcode Tools | Manage, create, and download barcodes. Also access barcode tools. | +| Genomic Selection | Can search for traits, start building a GS model, and predict values based on genotypes | +| **Sequence Analysis** | | +| BLAST | Sequence homology search | +| **Other** | | +| Ontology Browser | Browse all recorded ontologies | + +## Working with Lists {#working-with-lists} + +Lists are collections of identifiers that are stored in the database. Lists can be composed of accessions, plots, traits, locations, and trials. A given list can only contain one type of items, for example, all items have to be of type location in a list of locations. Lists are attached to the individual user's account, and can only be created and seen by the user while logged in, however, lists can be made public for all other users to see. SGN databases make heavy use of lists in a number of tools on the website. For example, trials are created using lists of accessions. + +### Creating lists + +Lists can be generated in various ways: + +One way to create a list is by clicking on the "Lists" link located on the toolbar. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_start.png') +``` + +To create a new list, enter the name of your new list and then click on the “New List” button. The name of the list can be anything, but should be unique and should be something to help you easily identify the list. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_new_list.png') +``` + +If the list already exists, it will appear on the “Your Lists” dialog. To add items to your list, click on the "View" icon to open the “List Contents” section. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_view_list.png') +``` + +On the “List Contents” page, enter items that you want to add to the list, then click on “Add” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_add_items.png') +``` + +The page will be updated and will display your items in a table at the bottom of the dialog It is possible to sort the list if you need. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_added_items.png') +``` + +Select the type of items in your list. To verify that the items that you added to your list are already stored in the database and that you selected a correct type for the items, click on the “Validate” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/list_manager_list_types.png') +``` + +If those items are already in the database, a message will indicate that “This list passed validation” + +```{r echo=FALSE, out.width='50%', fig.align='center'} +knitr::include_graphics('assets/images/image346.png') +``` + +Note that a list cannot contain duplicate elements. If a duplicate item is entered, the list manager will inform the user that the element is already in the list and will not add it again. + +Another easy way to create a list is to use the \@ref(search-wizard), which can be accessed from the Search menu. + +### Viewing and editing lists + +Lists can be viewed and edited using the "Lists" link on the toolbar. Clicking on the link will open a dialog that displays all of your lists, as well as an option to create new lists. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image71.png') +``` + +This page shows all lists that have been created, including those created by using the Search Wizard. You can view and edit your lists by using “Actions” buttons. + +1. Clicking on the "view" icon will open a new window called "List Contents" that allows you to change the list name, the type of the list, add new items, or delete existing items. + +2. Clicking on the “delete” icon will delete your list. **Caution: this action cannot be undone**. + +3. Clicking on the “download” icon will download the contents of your list to your computer. + +4. Clicking on the “make public” icon will make your list available for other users to view and use your list. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image297.png') +``` + +## User Permissions + +Breedbase accounts are assigned one or more of four different roles to determine the level of access they have within the database. The possible roles are **User**, **Submitter**, **Sequencer**, and **Curator**. Each role grants specific permissions, and careful management of them helps prevent data from being altered or deleted in error. + +```{r echo=FALSE, out.width='50%', fig.align='center'} +knitr::include_graphics('assets/images/roles.png') +``` + +Accounts are also assigned Breeding Program role(s) to grant access to the specfic breeding program(s) they work with. + +- The **User** role gives an account permission to view and download data throughout the database. +- The **Submitter** role gives an account permission to design field experiments and to upload and edit data using the tools in the “Manage” section. In order to submit and manage breeding data within a given breeding program, a submitter also must have a matching Breeding Program role. +- The **Sequencer** role gives an account permission to design genotyping experiments and submit plates to a genotyping service. +- The **Curator** role gives an account permission to do all of the above, as well as to delete data within the database. The Curator role also enables the addition or deletion of roles for all database accounts in the 'Manage User Roles' tool. diff --git a/docs/r_markdown_docs/build.R b/docs/r_markdown_docs/build.R new file mode 100644 index 0000000000..ad5e34a1df --- /dev/null +++ b/docs/r_markdown_docs/build.R @@ -0,0 +1,4 @@ +library(bookdown) +library(xfun) +is_abs_path <- FALSE +render_book("index.Rmd", "bookdown::gitbook") diff --git a/docs/r_markdown_docs/conditional-content.lua b/docs/r_markdown_docs/conditional-content.lua new file mode 100644 index 0000000000..8448e28ec2 --- /dev/null +++ b/docs/r_markdown_docs/conditional-content.lua @@ -0,0 +1,15 @@ +function Div(el) + if el.attributes['show-in'] then + if FORMAT:match 'latex' then + if el.attributes['show-in'] == "html" then + el.content = "" + return el + end + elseif FORMAT:match 'html' then + if el.attributes['show-in'] == 'pdf' then + el.content = "" + return el + end + end + end + end \ No newline at end of file diff --git a/docs/r_markdown_docs/data_analysis_tools.Rmd b/docs/r_markdown_docs/data_analysis_tools.Rmd new file mode 100644 index 0000000000..585aef013a --- /dev/null +++ b/docs/r_markdown_docs/data_analysis_tools.Rmd @@ -0,0 +1,639 @@ +# Data Analysis Tools + +SGN databases provides several tools for phenotype data analysis, marker-assisted selection, sequence and expression analyses, as well as ontology browser. These tools can be found in the “Analyze” menu. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image114.png") +``` + +## Selection Index + +To determine rankings of accessions based on more than one desirable trait, SGN databases provide a “Selection Index” tool that allows you to specify a weighting on each trait. To access the tool, clicking on “Selection Index” in the “Analyze” menu. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image251.png") +``` + +On the Selection Index page, selecting a trial that you want to analyze. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image95.png") +``` + +After you selected a trial, you can find traits that were assayed in that trial in the “Trait” box. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image78.png") +``` + +Selecting a trait that you want to include in the analysis will open a new dialogue showing the selected trait and a box that you can assign a “Weight” of that trait. After you are done, you can continue by selecting another trait by clicking on “Add another trait” link. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image304.png") +``` + +After you selected another trait, this page will automatically update information for you by showing all of the traits that you selected for the analysis. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image76.png") +``` + +You also have options to choose a reference accession, choose to include accessions with missing phenotypes, scaling values to a reference accession. After you complete your setting, clicking on “Calculate Rankings” + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image343.png") +``` + +The Selection Index tool will generate rankings of accessions based on the information that you specified. You can copy the results to your system clipboard, convert the table data to CSV format, or print the data. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image326.png") +``` + +Clicking on “Raw Average” will display average values of the phenotypes of those ranked accessions. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image150.png") +``` + +Selection Index tool also allows you to save top ranked accessions directly to “Lists”. You can retrieve top ranked accessions by selecting a number or a percent. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image156.png") +``` + +## Genomic Selection + +The prediction of breeding values for a trait is a one step or two steps process, depending on what stage in your breeding cycle you are. The first step is to build a prediction model for a trait using a training population of clones with phenotype and genotype data. If you have yet to select parents for crossing for your first cycle of selection you can use the breeding values of the training population. If you are at later stages of your selection program, you need to do the second step which is applying the prediction model on your selection population. All clones in your training and selection populations must exist in the database. + +To use the genomic selection tool, on [*cassavabase.org*](http://cassavabase.org/), select "Genomic Selection" from the "analyze" pull-down menu. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image247.png") +``` + +There are three ways to build a model for a trait. + +### Building a Model - Method 1: {#method-1} + +One way to build a model is, using a trait name, to search for trials in which the trait was phenotyped and use a trial or a combination of trials to build a model for the trait. For example, if you search for "mosaic disease severity, you will get a list of trials you can use as training populations. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image160.png") +``` + +You will get a list of trials (as shown below) in which the trait of your interested was phenotyped. From the list, you can use a single trial as a training population or combine several trails to form a training population for the prediction model of the trait. Let"s say, you want to create a training population using individuals from trials "cassava ibadan 2001/02" and "cassava ibadan 02/03" and build a model for "cassava mosaic disease severity" using all clones from the training population. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image249.png") +``` + +Select the trials to combine (the same coloured), click ‘done selecting’, click the "combine trials and build model" button, and you will get a model and its output for the trait. On the model detail page, you can view the description of input data used in the model, output from the model and search interface for selection populations the model you can apply to predict their breeding values. The description of the input data for the model includes the number of phenotyped clones, and the number of markers, scatter and frequency distribution plots for the phenotype data, relationship between the phenotype data and GEBVs, population structure. The model output includes model parameters, heritability of the trait , prediction accuracy, GEBVs of the individuals from the training population and marker effects. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image330.png") +``` + +Expand each section to see detailed information. + +If you expand the ‘Trait phenotype data’ section, you will find plots to explore the phenotype data used in the model. You can assess the phenotype data using a scatter and histogram plots and the descriptive statistics. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image244.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image263.png") +``` + +A regression line between observed phenotypes and GEBVs shows the relationship between the two. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image83.png") +``` + +You can also explore if there is any sub-clustering in the training population using PCA. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image93.png") +``` + +To check the model accuracy, a 10-fold cross-validation test, expand the ‘model accuracy’ section. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image328.png") +``` + +Marker effects are also available for download. To do so, expanad the ‘Marker Effects’ section and click the ‘Download all marker effects’ link and you will get a tab delimited output to save on your computer. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image74.png") +``` + +The breeding values of the individuals used in the training population are displayed graphically. Mousing over each data point displays the clone and its breeding value. To examine better, you can zoom in into the plot by selecting an area on the plot. You can download them also by following the "Download all GEBVs" link. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image147.png") +``` + + +#### Estimating breeding values in a selection population {-} + +If you already have a selection population (in the database), from the same model page, you can apply the model to the selection population and estimate breeding values for all the clones in the population. You can search for a selection population of clones in the database using the search interface or you can make a custom list of clones using the [*list interface*](#working-with-lists). If you click the "search for all relevant selection populations", you will see all relevant selection populations for that model. However, this option takes long time decause of the large set of populations in the database and the filtering. Therefore, the fastest way is to search for each of your selection populations by name. If you are logged in to the website you will also see a list of your custom set of genotyped clones. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image338.png") +``` + +To apply the model to a selection population, simply click your population name or "Predict Now" and you will get the predicted breeding values. When you see a name of (or acronym\]) of the trait, follow the link and you will see an interactive plot of the breeding values and a link to download the breeding values of your selection population. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image334.png") +``` + +### Building a Model - Method 2 {#method-2} + +Another way to build a model is by selecting a trial, instead of selecting and searching for a specific trait. This approach is useful when you know a particular trial that is relevant to the environment you are targeting to breed material for. This method allows you to build models and predict genomic estimated breeding values (GEBVs) for several traits within a single trial at once. You can also calculate selection index for your clones when GEBVs are estimated for multiple traits. + +To do this select the "Genomic Selection" link found under the "analyze" menu. This will take you to the same home page as used with Method 1. However, instead of entering information to search for in "Search for a trait", click on "Use a trait as a trial population". This will expand a new menu that will show all available trials. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image344.png") +``` + +```{r echo=FALSE, out.width="25%", fig.align="center"} +knitr::include_graphics("assets/images/image329.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image341.png") +``` + +To begin creating the model, select the existing trial that you would like to use. In this example I will be using the trial and trait data from "Cassava Ibadan 2002/03" trial. Clicking on a trial will take you to a page where you can find information such as number of markers and number of phenotypes clones. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image322.png") +``` + +In addition to the number of phenotype clones and number of markers, the main page for the trial selected also has information and graphs on phenotypic correlation for all of the traits. By moving your cursor over the graph you can read the different values for correlation between two traits. A key with all of the trait names of the acronyms used can be found in the tab below the graph. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image151.png") +``` + +Below the "Training population summary" there is a tab for "Traits". Clicking on this tab will show all available traits for the specific trial. You can create a model by choosing one or multiple traits in the trial and clicking "Build Model". In this example, the traits for "cassava bacterial blight severity" and "cassava mosaic disease severity" have been selected. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image69.png") +``` + +Clicking on "Build Model" will take you to a new page with the models outputs for the traits. Under the "Genomic Selection Model Output" tab you can view the model output and the model accuracy. Clicking on any of the traits will take you to a page with information about the model output on that individual trait within the trial. There you can view all of the trait information that was seen in more detail in [*Method 1*](#method-1). + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image336.png") +``` + +You can apply the models to simultaneously predict GEBVs for respective traits in a selection population by clicking on "Predict Now" or the name of the selection population. You can also apply the models to any set of genotyped clones that you can create using the "lists" feature. For more information on lists, click [*here*](#working-with-lists). Follow the link to the trait name to view and download the predicted GEBVs for the trait in a selection population. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image171.png") +``` + +To compare clones based on their performance on multiple traits, you can calculate selection indices using the form below. Choose from the pulldown menu the population with predicted GEBVs for the traits and assign relative weights for each trait. The relative weight of each trait must be between 0 - 1. 0 being of least weight and importance, not wanting to consider that particular trait in selecting a genotype and 1 being a trait that you give highest importance. + +In this example we will be using the "Cassava Ibadan 2002/03" population and assigning values to each of the traits. Remember that there is a list of acronyms and trait names at the bottom of the page for reference. After entering whatever values you would like for each trait click on the "Calculate" button to generate results. This will create a list of the top 10 genotypes that most closely match the criteria that you entered. The list will be displayed right below the "selection index" tab. This information can also be downloaded onto your computer by clicking on the "Download selection indices" link underneath the listed genotypes and selection indices. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image81.png") +``` + +### Building a Model - Method 3 + +In addition to creating a model by searching for pre-existing traits or by preexisting trial name, models can also be created by using your own list of clones. This creates a model by using or creating a training population. + +The page to use the third Method for creating a population model is the same as for the other two models. Select "Genomic Selection" from under the "analyze" menu of the main toolbar. This will take you to the Genomic Selection homepage and show you all three available methods to create a model. To see and use Method 3 scroll down and click on the tab labeled "Create a Training Population". This will open a set of tools that will allow you to use pre-existing lists or to create a new list. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image138.png") +``` + +Once the "Create a Training Population" tab is opened you have the option to use a pre-existing list or create new one. To learn how to create a list, click [*here*](#working-with-lists). The "Make a new list of plots" link will take you directly to the Search Wizard that is usually used to create lists. + +Please note: the only lists that can be used in Method 3 to create a model are lists of plots and trials. If the pre-existing list is not of plots or trials (for example, traits, or locations) it will not show up and cannot be used as a training population. When you create you use a list of trials, the trials data will be combined to create a training data set. + +To use your custom list of plots or trials as a training population, select the list and click "Go". This will take you to a detail page for the training population. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image181.png") +``` + +From here on you can build models and predict breeding values as described in [*Method 2*](#method-2)**.** + +## Genome Browsing + +There are two ways to evaluate genotype information within the browser, from an accession detail page or a trial detail page. + +### Browsing Genotype data by Accession + +If you are interested in browsing genotype information for a single accession, for example ‘BAHKYEHEMAA’, navigate to the accession detail page. + +```{r echo=FALSE, out.width="50%", fig.align="center"} +knitr::include_graphics("assets/images/image152.png") +``` + +Near the bottom of the detail page is a collapsible section called “Accession Jbrowse”. + +```{r echo=FALSE, out.width="50%", fig.align="center"} +knitr::include_graphics("assets/images/image20.png") +``` + +This section will contain a link to the accession jbrowse page if the necessary genotype data is available. Clicking the link should take you to a page that looks like this, a which point you can browsre the genotype data in the form of a vcf track aligned to the latest build of the genome. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image318.png") +``` + +### Browsing Genotype data by Trial + +If you are interested in browsing genotype information for the accessions within a given trial, navigate to the trial detail page. + +```{r echo=FALSE, out.width="50%", fig.align="center"} +knitr::include_graphics("assets/images/image277.png") +``` + +Halfway down the page is a collapsible section called “Trial Jbrowse”. This section will contain a link to the trial jbrowse page if the necessary genotype data for at least two accessions planted in the trial is available. + +```{r echo=FALSE, out.width="50%", fig.align="center"} +knitr::include_graphics("assets/images/image268.png") +``` + +Clicking the link should take you to a page that looks like this, a which point you can browse the genotype data in the form of vcf tracks aligned to the latest build of the genome. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/image327.png") +``` + +## Principal Component Analysis (PCA) + +Principal component analysis helps estimate and visualize if there is sub-grouping of individuals within a dataset based on a number of variables. Currently, you can use marker data to run PCA on datasets. + +You can run PCA from multiple places on the website. To do PCA on + +(1) individuals from a trial, go to the trial detail page and find the PCA tool under the "Analysis tools" section. + +(2) individuals from a training population you used in a GS modeling, do your modeling and find the PCA tool in the model output page. + +(3) individuals in a training population and selection population you applied the training model, do your modeling, apply the model on the selection population and find the PCA tool on the selection population prediction output page. + +(4) individuals in a list of accessions you created, for example using the search wizard, go to the "Analyze" menu and select the "Population Structure", select your list of individuals and run PCA. + +(5) individuals from multiple trials, create a list of the trials using the search wizard, go to the "Analyze" menu and select the "Population Structure", select your list of trials and run PCA. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/pca_iita_naccri_trials.png") +``` + +With all the options, you will get a interactive plot of the two PCs (shown below) that explain the largest variance. Point the cursor at any data point and you will see the individual name with its corresponding PCs scores. By clicking the ‘Download all PCs’, you can also download the 10 PCs scores in the text format. + + +## ANOVA + +Currently, ANOVA is implemented for a single trial (single year and single location). You can do ANOVA for RCBD, CRD, Alpha and Augmented trial designs. ANOVA is done using linear mixed effects model, where the genotypes is fixed effect and the replications and blocks are random effects. Fixed effect significance level is computed using "lmer" from "lmeTest" R package. + +You can do ANOVA from two places: trial detail and training population detail. In both cases, if the phenotype data was from the supported trial designs, + +-- Go to the ANOVA section down in the trial or training population page + +-- Select the trait of you want to perform ANOVA + +-- Click the "Run ANOVA" and wait for the result + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/anova-dm.png") +``` + +## Clustering (K-Means, Hierarchical) + +The K-Means method allows you to partition a dataset into groups (K number). The hierarchical clustering, agglomerative, allows you to explore underlying similarity and visualize in a tree structure (dendrogram) the different levels of similarities (clusters) among samples. You can do clustering based on marker data, phenotype data and GEBVs. When you use phenotype data, first clone averages for each trait are calculated. Both methods use Euclidean distance as a measure of similarity. For the hierachical clustering, the complete-linkage (farthest neighbour) method is used to link up clusters. + +There are three pathways to using this tool. + +(1) When you have data in the form of a list or dataset from the search wizard: + + (A) -- go to the "Analyze" menu and select the clustering option + + (B) -- make sure you are logged in + + (C) -- Select the relevant genotyping protocol, if you are clustering using genotype data + + (D) -- select your list or dataset, click "Go" + + (E) -- select clustering type + + (F) -- select the data type to use + + (G) -- If you are running K-Means clustering, provide the number of partitions (K). If left blank it will partition the data set into optimal numbers for the dataset. + + (H) -- click the "Run Cluster" and wait for the analysis to finish or queue the request and wait for an email with the analysis result. + + (I) -- You can download the outputs following the download links. + +(2) From the trial detail page: + + (A) -- Go to the "Analysis Tools" section + + (B) -- Follow steps D to G in (1) + +(3) In the solGS pipeline: + + (A) -- Once you you are in a model output put page, you will see a section where you can do clustering in the same way as above (option 2). + +K-Means clustering: + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/k-means-cluster.png") +``` + +Hierarchical clustering: + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/hclustering.png") +``` + +## Genetic Gain + +You can check for genetic gain by comparing the the GEBVs of a training and a selection population. You can do this in the solGS pipepline once you build a model and apply the model to predict the GEBVs of a selection population. Once at that stage, you will see a section "Check Genetic Gain". Select a selection population to compare with the training population and click the "Check Genetic Gain" button. The genetic gain will be visualized in boxplots. You can download the boxplot(s) as well as the GEBVs data used for the plot(s). + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/genetic-gain.png") +``` + + +## Kinship and Inbreeding Coefficients + +This tool allows you to estimate genetic relatedness between a pair of individuals (kinship), homozygousity across loci in an individual (inbreeding coefficient), and genetic similarity of an individual relative to the rest of the population (averge kinship). + +There are three pathways to using this tool. + + (1) When you have a list or dataset clones, created from the search wizard: + + (A) -- go to the "Analyze" menu and select the kinship and inbreeding + + (B) -- make sure you are logged in + + (C) -- Select the genotypic protocol for the marker data + + (D) -- select your list or dataset of clones, click "Go" + + (F) -- click the "Run Kinship" and wait for the analysis to finish, depending on the data size this may take minutes. You can choose to submit the analysis and wait for an email notice to view the results or wait for it to complete. + + (G) -- You can download the output following the download links. + +(2) From the trial detail page: + + (A) -- Go to the "Analysis Tools" section + + (B) -- Follow steps C to G in (1) + +(3) In the solGS pipeline: + + (A) -- Once you you are in a model output put page, scroll down to the "Kinship and Inbreeding" section and run kinship. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/kinship-inbreeding.png") +``` + +## Creating Crossing Groups + +If you calculate selection index based on GEBVs of multiple traits, and you want to select a certain proportion of the indexed individuals (e.g. top 10%, or bottom 10%) and then you want to partition the selected individuals into a number of groups based on their genotypes, you can use the k-means clustering method. + +The procedure is: + +(1) predict GEBVs for 2 or more traits + +(2) In the models output page, calculate selection indices. Note the name of the selection index data. + +(3) Go to the clustering section, + + -- select the selection index data, + + -- select "K-means", + + -- select "Genotype", + + -- in the K-numbers textbox, fill in the number of groups you want to create, + + -- in the selection proportion textbox, fill in the proportion of the indexed individuals you want to select, e.g. for the top 15 percent, 15. if you wish to select bottom performing, prefix the number with minus sign (e.g. -15) + + -- then run cluster and wait for the result. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/selection_proportion_clustering.png") +``` + +## Search Wizard Genomic Relationship Matrix (GRM) Download + +The genomic relationship matrix (GRM) is useful for understanding underlying structure in your population. Breedbase can compute the GRM using rrBLUP. First, select accessions in the search wizard and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. The GRM can be returned in a matrix format (.tsv) which shows all pairwise relationships between the selected accessions and is useful for visualization; alternatively, the GRM can be returned in a three-column format (.tsv) which is useful for programs like ASReml outside of Breedbase. The GRM can also be returned as a simple correlation heatmap image (.pdf). The GRM can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox "compute from parents"; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/search_wizard_genotype_analyses_grm.png") +``` + + +## Search Wizard Genome Wide Association Study (GWAS) + +Performing a genome wide association study (GWAS) can determine genotypic markers which are significantly correlated to phenotypic traits. Breedbase can compute GWAS using rrBLUP. First, select accessions and trait(s) in the search wizard, and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Several traits can be selected in the search wizard; if the traits are not to be treated as repeated measurements then select "no" in the select box and this will tell Breedbase to return GWAS results independently for the selected traits. If the selected traits are indeed all repeated measurements then select "yes" in the select box and Breedbase will return as single GWAS analysis across all the phenotypic records. Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. GWAS results can be returned in a tabular format (.tsv) where the -log10(p-values) for the selected traits are returned; alternatively, the GWAS results can be returned as Manhattan and QQ plots for the selected traits. The GWAS can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox "compute from parents"; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. + +The GWAS will filter the data by the input MAF and missing data filters provided. After filtering the data is imputed using an "EM" method in rrBLUP. The Kinship matrix (GRM) is computed from the imputed genotypic data and used in the GWAS model. The GWAS uses fixed effects for different field trials and replicates in the phenotypic data. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/search_wizard_genotype_analyses_gwas.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/search_wizard_genotype_analyses_manhattan_plot.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/search_wizard_genotype_analyses_qq_plot.png") +``` + + +## Spectral Analysis {#spectral-analysis} + +Visible and near-infrared spectroscopy (vis-NIRS) can be related to reference phenotypes through statistical models to produce accurate phenotypic predictions for unobserved samples, increasing phenotyping throughput. This technique is commonly used for predicting traits such as total starch, protein, carotenoid, and water content in many plant breeding programs. Breedbase implements the R package [*waves*](https://CRAN.R-project.org/package=waves) to offer training, evaluation, storage, and use of vis-NIRS prediction models for a wide range of spectrometers and phenotypes. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/waves_breedbase_schema.png") +``` + +### Dataset selection +In order to initiate an analysis, the user must select one or more datasets using \@ref(search-wizard). A dataset in Breedbase can contain observationUnit-level (plot-, plant-, or sample-level) trial metadata and phenotypic data from one or more trials. After navigating to the “NIRS” webpage under the “Manage” tab in Breedbase, the user can initiate an analysis and select one of these datasets as input for model training. An optional test dataset can be selected in the second step of the workflow. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_prediction_workflow_intro.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_prediction_workflow_dataset.png") +``` + +### Cross-validation +Five cross-validation schemes that represent scenarios common in plant breeding are available for this analysis. These include CV1, CV2, CV0, and CV00 as outlined below and described in depth by Jarquín et al. (2017) as well as random and stratified random sampling with a 70% training and 30% validation split. For those schemes from Jarquín et al. (2017), specific input datasets must be chosen based on genotype and environment relatedness. Cross-validation choices: +* **Random sampling** (70% training / 30% validation) +* **Stratified random sampling**, stratified based on phenotype (70% training / 30% validation) +* **CV1**, untested lines in tested environments +* **CV2**, tested lines in tested environments +* **CV0**, tested lines in untested environments +* **CV00**, untested lines in untested environments + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_cv.png") +``` + +### Preprocessing +Preprocessing, also known as pretreatment, is often used to increase the signal to noise ratio in vis-NIR datasets. The *waves* function *DoPreprocessing()* applies functions from the *stats* and *prospectr* packages for common spectral preprocessing methods with the following options: +* Raw data (default) +* First derivative +* Second derivative +* Gap segment derivative +* Standard normal variate (SNV; Barnes et al., 1989) +* Savitzky-Golay polynomial smoothing (Savitzky and Golay, 1964) + +For more information on preprocessing methods and implementation, see the [*waves*](https://CRAN.R-project.org/package=waves) manual, available through CRAN: [waves.pdf](https://cran.r-project.org/web/packages/waves/waves.pdf) + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_snv.png") +``` + +### Algorithms +Several algorithms are available for calibration model development in Breedbase via the [*waves*](https://CRAN.R-project.org/package=waves) package. The *TrainSpectralModel()* function in waves performs hyperparameter tuning as applicable using these algorithms in combination with cross validation and train functions from the package *caret*. Currently, only regression algorithms are available, but classification algorithms such as PLS-DA and SVM clasification are under development. +* **Partial least squares regression** (PLSR; Wold et al., 1982; Wold et al., 1984) is a popular method for spectral calibrations, as it can handle datasets with high levels of collinearity, reducing the dimensionality of these data into orthogonal latent variables (components) that are then related to the response variable through a linear model (reviewed in Wold et al., 2001). To avoid overfitting, the number of these components included in the final model must be tuned for each use case. The PLSR algorithm from the *pls* package is implemented by waves. +* **Random Forest regression** (RF; Ho, 1995) is a machine learning algorithm based on a series of decision trees. The number of trees and decisions at each junction are hyperparameters that must be tuned for each model. Another feature of this algorithm is the ability to extract variable importance measures from a fitted model (Breiman, 2001). In Breedbase, this option is made available through implementation of the RF algorithm from the package randomForest in the waves function TrainSpectralModel(). This function outputs both model performance statistics and a downloadable table of importance values for each wavelength. It is worth noting that this algorithm is computationally intensive, so the user should not be alarmed if results do not come right away. Breedbase will continue to work in the background and will display results when the analysis is finished. +* **Support vector machine regression** (SVM; Vapnik, 2000) is another useful algorithm for working with high-dimension datasets consisting of non-linear data, with applications in both classification and regression. The package waves implements SVM with both linear and radial basis function kernels using the kernlab package. + +### Output: common model summary statistics +After training, model performance statistics are both displayed on a results webpage and made available for download in .csv format. These statistics are calculated by the *TrainSpectralModel()* function in [*waves*](https://CRAN.R-project.org/package=waves) using the *caret* and *spectacles* packages. Reported statistics include: +* Tuned parameters depending on the model algoritm + * **Best.n.comp**, the best number of components to be included in a PLSR model + * **Best.ntree**, the best number of trees in an RF model + * **Best.mtry**, the best number of variables to include at every decision point in an RF model +* **RMSECV**, the root mean squared error of cross-validation +* **R2cv**, the coefficient of multiple determination of cross-validation for PLSR models +* **RMSEP**, the root mean squared error of prediction +* **R2p**, the squared Pearson’s correlation between predicted and observed test set values +* **RPD**, the ratio of standard deviation of observed test set values to RMSEP +* **RPIQ**, the ratio of performance to interquartile distance +* **CCC**, the concordance correlation coefficient +* **Bias**, the average difference between the predicted and observed values +* **SEP**, the standard error of prediction +* **R2sp**, the squared Spearman"s rank correlation between predicted and observed test set values + +### Export model for later use +Once a model has been trained, it can be stored for later use. This action calls the *SaveModel()* function from [*waves*](https://CRAN.R-project.org/package=waves). Metadata regarding the training dataset and other parameters specified by the user upon training initialization are stored alongside the model object itself in the database. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_export_model.png") +``` + +### Predict phenotypes from an exported model (routine use) +For phenotype predictions, users select a dataset and can then choose from models in the database that were trained using the same spectrometer type as the spectral data in the chosen dataset. Predicted phenotypes are stored as such in the database and are tagged with an ontology term specifying that they are predicted and not directly measured. Metadata regarding the model used for prediction is stored alongside the predicted value in the database. Predicted phenotypes can then be used as normal in other Breedbase analysis tools such as the Selection Index and GWAS. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_select_model.png") +``` + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/manage_NIRS_prediction_results.png") +``` + +### FAQ +The Breedbase Spectral Analysis Tool does not allow for prediction models involving data from multiple spectrometer types at once. + +References +* Barnes, R.J., M.S. Dhanoa, and S.J. Lister. 1989. Standard normal variate transformation and de-trending of near-infrared diffuse reflectance spectra. Appl. Spectrosc. 43(5): 772-777. doi: 10.1366/0003702894202201. +* Breiman, L. 2001. Random forests. Mach. Learn. 45: 5-32. doi: 10.1201/9780429469275-8. +* Ho, T.K. 1995. Random decision forests. Proc. Int. Conf. Doc. Anal. Recognition, ICDAR 1: 278-282. doi: 10.1109/ICDAR.1995.598994. +* Jarquín, D., C. Lemes da Silva, R.C. Gaynor, J. Poland, A. Fritz, et al. 2017. Increasing Genomic-Enabled Prediction Accuracy by Modeling Genotype x Environment Interactions in Kansas Wheat. Plant Genome 10(2): plantgenome2016.12.0130. doi: 10.3835/plantgenome2016.12.0130. +* Johnson, R.A., and D.W. Wichern. 2007. Applied Multivariate Statistical Analysis (6th Edition). +De Maesschalck, R., D. Jouan-Rimbaud, and D.L. Massart. 2000. The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. doi: 10.1016/S0169-7439(99)00047-7. +* Mahalanobis, P.C. 1936. On the generalized distance in statistics. Natl. Inst. Sci. India. +* Savitzky, A., and M.J.E. Golay. 1964. Smoothing and Differentiation of Data by Simplified Least Squares Procedures. Anal. Chem. 36(8): 1627-1639. doi: 10.1021/ac60214a047. +* Shrestha, R., L. Matteis, M. Skofic, A. Portugal, G. McLaren, et al. 2012. Bridging the phenotypic and genetic data useful for integrated breeding through a data annotation using the Crop Ontology developed by the crop communities of practice. Front. Physiol. 3 AUG(August): 1-10. doi: 10.3389/fphys.2012.00326. +* Vapnik, V.N. 2000. The Nature of Statistical Learning Theory. Springer New York, New York, NY. +* Wold, S., A. Ruhe, H. Wold, and W.J. Dunn, III. 1984. The Collinearity Problem in Linear Regression. The Partial Least Squares (PLS) Approach to Generalized Inverses. SIAM J. Sci. Stat. Comput. 5(3): 735-743. doi: 10.1137/0905052. +* Wold, S., M. Sjöström, and L. Eriksson. 2001. PLS-regression: a basic tool of chemometrics. Chemom. Intell. Lab. Syst. 58(2): 109-130. doi: 10.1016/S0169-7439(01)00155-1. + + +## General Mixed Model Tool + +The general mixed model tool is available at /tools/mixedmodels and a link is provided from the Analyze menu. + +To use the mixed model tool, first create dataset using the Wizard containing the data that you would like to analyze. + +Select the Mixed Model tool from the Analyze menu. + +You are presented with a workflow. On the first step of the workflow, select the dataset that you wish to analyze, click on "Choose dataset" to continue. + +The second part of the workflow presents you with the traits in the dataset; you can select one or more traits from the lists using the select buttons. If you selected one trait, a bargraph of the trait distribution will be shown. Click the "Next step" button to move to the next screen. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/mixedmodel_tool_model_build_step.png") +``` + +On the model build screen, all the factors are displayed that are contained within the dataset. The factors are presented as a list of blue buttons that can be dragged using the mouse to areas on the screen which build a mixed model equation. The areas correspond to fixed factors, random factors, and optionally to more complex factors, such as fixed factors with interaction and fixe factors with vriable slope/intersects. Drag the available factors to the corresponding area. To calculate BLUPs for germplasm, drag the germplasmName button to the "Random factors" area. To calculate BLUEs, drag it to the "Fixed factors" area. The factors need to have different levels contained within them, for example, if there is only one trial in the dataset, it cannot be used as one of the factors. Click on "Run analysis and got to next step" to run the mixed model and display the results. + +The result view contains two tabs, one with the raw data, either BLUPS or BLUEs, and the other the adjusted means from the raw data. + +The results can be stored in the database as an analysis, by clicking the button provided on the top of the data. + + +## Genomic Prediction of Cross Performance (GPCP) + +The GPCP tool is available at /tools/gcpc and a link is provided from the Analyze menu. +The GCPC tool implements genomic prediction with additive and directional dominance in the linear mixed model to predict for cross performance. + +Before using the tool, first create a dataset using the Wizard containing the data that you would like to analyze. (The dataset should have genotyping_protocols). +Second, create Selection Indices for your traits using Selection Index in Analyze Menu. + +To use the tool, Select the GPCP tool from the Analyze menu. + +Then, select the dataset with genotyping_protocols that you wish to analyze, click on "Proceed to Factor Selection" to load available factors that can be included in the model. + +Select the factors you wish to include in the model either as Fixed or Random. Click "None" for factors that you don"t want to include in the model. Note that the "germplasmName" is factored as Random by default. + +The next step is to select the selection index for your traits on the dropdown menu. + +Once you are through, click "Run GPCP" to run the model. The output will be presented in form of a table with "ID", "Parent1", "Parent2" and their cross prediction merit organized in descending order. +The results will also have sex information based on whether the dataset has plant sexes available in the database. + +## Tool Compatibility + +The dataset definition enables one to predict whether the dataset can be used in various analysis tools. + +Upon creating a dataset, the site will automatically predict its compatibility with the available analysis tools and report these values on the dataset details page. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/tool_compatibility_details.png") +``` + +In the table, each tool will report to the user which traits are available to be analyzed based on phenotype data, and if different types of analyses are available, these will also be reported to the user. Some tools may give a warning sign to indicate that this dataset is compatible, but with potentially low sample sizes. Hover over the warning symbol to get a readout of the reason for the warning. + +Below the table, there is a button that enables the user to re-calculate tool compatibility. This can be useful if a dataset is created before phenotypes are uploaded to a trial, since phenotype data is used in determining dataset compatibility. Even if the page appears to hang, do not worry; the compatibility check will continue in the background, and you can check later. + +```{r echo=FALSE, out.width="95%", fig.align="center"} +knitr::include_graphics("assets/images/tool_compatibility_data_summary.png") +``` + +Below the tool compatibilities, there is also a summary of the data encompassed by the dataset and the criteria used for determining tool compatibility. Those criteria are used in the following way: +- Correlation: A dataset can be used in a correlation analysis if there are many phenotype measurements for different traits made on the same accession. +- Population Structure (PCA): A genotype PCA can be run if there are many accessions all genotyped with the same protocol. A phenotype PCA can be run if many accessions all have measurements on many traits. +- Clustering: Like a PCA, clustering can be done in both phenotype and genotype modes. They have the same requirements as PCA. +- Kinship & Inbreeding: A dataset with many accessions genotyped with the same protocol can be used for kinship analyses. +- Stability: A dataset containing many accessions with the same trait measured across multiple locations can be used in stability analyses. +- Heritability: This requires one or more trials with the same trait measured on the same accession across those trial(s). +- Mixed Models: This requires sufficient accession numbers, trait measurements, and trial designs. +- GWAS: A dataset is compatible with GWAS if there are many accessions genotyped for the same genotyping protocol, and the genotyping protocol has enough markers to run a GWAS. In addition, each accession needs to be phenotyped for a trait. +- Boxplotter: There must be sufficient trait measurements to make a boxplot of the trait. + +In addition to being on the dataset details page, tool compatibilities may be listed on the dataset selection screens for analysis tools. The compatibilities are non-blocking; you may always try using a dataset in an analysis even if there are warnings or if it is deemed non-compatible. As before, you can hover over the warning symbols to see why a dataset may not have statistical power. For analyses with multiple modes, such as clustering and PCA, you can also hover over the compatibliity checkmark to see what types (phenotype or genotype) the dataset is compatible with. diff --git a/docs/r_markdown_docs/hypothesis.html b/docs/r_markdown_docs/hypothesis.html new file mode 100644 index 0000000000..a577566c14 --- /dev/null +++ b/docs/r_markdown_docs/hypothesis.html @@ -0,0 +1 @@ + diff --git a/docs/r_markdown_docs/index.Rmd b/docs/r_markdown_docs/index.Rmd new file mode 100644 index 0000000000..fbc0c4423a --- /dev/null +++ b/docs/r_markdown_docs/index.Rmd @@ -0,0 +1,40 @@ +--- +title: "Breedbase User Manual" +author: "Breedbase team" +date: "`r Sys.Date()`" +knit: "bookdown::render_book" +graphics: yes +colorlinks: yes +fontsize: 12pt +description: "A manual for all things Breedbase. How to create an account, search the database, manage different data types, and analyze data." +url: 'https://solgenomics.github.io/' +github-repo: solgenomics/sgn +cover-image: assets/images/Breedbase_HighRes.png +site: bookdown::bookdown_site +--- + +# Introduction {.unnumbered} + +Welcome to the Breedbase manual! + +::: {show-in="html"} +Use the table of contents in the left sidebar to navigate to the topic of your choice. + +At any time you can select specific text in the manual to highlight or annotate it using [Hypothesis](https://web.hypothes.is/). Open the Hypothesis sidebar on the right to view existing annotations. + +You may also use the widgets at the top of the screen to: + + - collapse the sidebar
    - search for a specfic topic
    - change the font size, font type, or the site theme
    - download the manual as a pdf + +Manual as a pdf can be download here also. [ Download](BreedbaseManual.pdf) +::: + +This manual is intended for database users.\ +If you are a developer looking for software implementation details, please visit the developer wiki instead: + +````{=html} + +```` diff --git a/docs/r_markdown_docs/managing_accessions.Rmd b/docs/r_markdown_docs/managing_accessions.Rmd new file mode 100644 index 0000000000..d4aa012d24 --- /dev/null +++ b/docs/r_markdown_docs/managing_accessions.Rmd @@ -0,0 +1,212 @@ +--- +editor_options: + markdown: + wrap: sentence +--- + +# Managing Accessions + +The “Manage Accession” page provides links for adding new accessions. +New accessions can be added to the database by either using a List or by uploading an Excel file (either XLS or XLSX format). +Both options are explained in more detail below. +To begin, click on the "Add Accessions or Upload Accession Info" link. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_link.png') +``` + +This will open a dialog allowing you to select either "Using Lists" or "Uploading a File". + +## Add Accessions Using A List + +First we will show how to add accessions "Using Lists". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_using_lists.png') +``` + +Here you select an accession list which you have previously made (see List Manager chapter). +If you need to create or edit your list you can do so now by clicking "Manage Lists". +After selecting your list, click "Continue". + +The contents of the list will be checked against the database, and elements that are already present will be flagged. +A dialog will appear that will show the accessions which already exist in the database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_found.png') +``` + +After clicking on the "Continue" button, the next dialog will appear with accessions that have very similar names as the accession that you are adding. +In the example below, there are two accession with very similar names to accessions already in the database. +'TME0419' is very similar to 'TME419', and probably represent the same line, so it would be a mistake to add this the database again. +Duplicate lines in the database should be avoided, as they cause problems when evaluating lines; data is divided up among several duplicates, making it harder to get the full picture about an accession. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_fuzzy.png') +``` + +To avoid situations in adding a mistaken duplicate accession, the database gives you options for moving forward with these very similar looking accession names. +You can either "continue saving the name in your list", "replace name in your list with selected existing name", "remove name in your list and ignore", or "add name in your list as a synonym to selected existing name". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_fuzzy_options.png') +``` + +Clicking "Download Fuzzy Matches" will return a tabular result of the "fuzzy" accession name results shown. +Click "Make changes and continue" to move on. + +The final dialog shows the accessions that will be added. +Here you need to assign the species of these accessions. +You can optionally group the accessions into a population and/or add an organization for the accessions. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_complete_using_list.png') +``` + +Once you click "Add Accessions", the new accessions will be created in the database and you will see the following confirmation dialog, which includes links to the newly created accessions. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_saved.png') +``` + +## Uploading Accessions and Accession's Info From A File + +Uploading accessions using a file is very similar to using a list, but enables you to add a variety of attributes, such as synonyms or ploidy levels, to the accessions in bulk. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_using_file.png') +``` + +Clicking on "Spreadsheet format" will show the required structure of the spreadsheet. +The file must be XLS or XLSX format and can contain a number of header columns as attributes. +It is important that you use exactly the same header column names as listed here. +In columns that indicate that many attribute values can be passed at once using (s), such as synonym(s), you can pass a comma separated list of values, such as 'synonym1,synonym2'. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_spreadsheet_info.png') +``` + +Once you have selected your XLS or XLSX file for upload, click "Continue". + +The following process is the same way as with lists: + +The first dialog which can appear will show accession names which are already in the database. + +Click "Continue" and the next dialog that can appear will show "fuzzy" matches for the accession names you are trying to upload. +Here you can choose to prevent adding accession names which look very similar to each other as wrongly duplicated accessions. + +Click "Continue" and the final dialog that will appear will show the information to be added into the database. +Here it is divided into accession names that are new and accession names that already exist in the database; however, for the accession names that already exist it will show additional attributes that originated from your file that will be added to these accessions. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_complete_using_file.png') +``` + +Once you click "Add Accessions", the new accessions and information will be created in the database and you will see the following confirmation dialog, which includes links to the created and updated accessions. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/manage_accessions_add_accessions_saved.png') +``` + +## Email alert for accession upload + +When uploading a large number of accessions from a file, uploads can take a while, as the system needs to perform a series of checks on each entry. +You have the option to receive an email notification about the status and results of your upload by clicking the "Email Alert" checkbox. +By default, the system will use the email address associated with your account, but you have the option of entering a different email address if you prefer. +After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. +Once the process completes, you will receive an email with the upload results, including any warnings or errors that may have occurred during the upload. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/accession_upload_using_email.png') +``` + +## Add Parentage (Pedigree) Information to Accessions + +Pedigree data can be uploaded from your computer by clicking on “Upload Pedigree File” + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image286.png') +``` + +You can find detailed information on how to prepare pedigree file by clicking on “File format information”. +The file format can be tab or comma delimited text file, or Excel files (.xls or .xlsx). + +The currently supported format has four columns: + +progeny name female parent accession male parent accession type + +Type can be biparental, self, backcross, sib, polycross, reselected, or open. +In the case of the open type, the male parent accession field can remain blank. +For all other types, both columns should be filled, even if they contain the same information as another column (such as self). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image333.png') +``` + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/pedigree_upload_format.png') +``` + +## Working with grafts + +Grafts are plants that are composed of a rootstock and a scion, which are genetically different and fused together, usually at the stem level. + +To work with grafts, the grafts interface needs to be activated by a system administrator. +Please contact your Breedbase provider. +Briefly, a configuration parameter needs to be added to the sgn_local.conf file, show_grafting_interface. +It should be set to 1 in sgn_local.conf, the default is 0 in sgn.conf. + +Grafts to be created need to be specified using an Excel file (xlsx format) with two columns. +The first column should have the header "scion accession" and should list accession names that will be scions. +The second column should have the header "rootstock accession" and should list accession names that will be rootstocks. + +In the database, the graft accessions will be created as single accessions. +The graft accession will have two relationships, one to the scion accession (scion_of relationship) and one to the rootstock (rootstock_of relationship). +These relationships are displayed on the pedigree viewer. +The graft accession name is created from the scion accession name and the rootstock accession name, separated by the graft separator character. +By default, the graft separator character is the plus sign '+'. +The graft separator character can be changed in the sgn_local.conf file, using the parameter graft_separator_string. +The graft separator string should not occur in any other accession names that are not grafts. + +When the grafting interface is activated, a new button will be shown on the manage accessions page, called "Upload Grafts". + +Clicking the button brings up the upload grafts dialog. + +Select the Excel file containing the grafting information. +The system will validate the file, for example, check whether the accessions are in the database, and if the headers are correct. + +The validation result will be presented, and if problems are found, they will be listed. +In addition, if there are problems, the Upload button will be grayed out and upload will not be possible. +Conversely, if there are no problems, the Upload button will be activated and can be clicked to store the data. + +If the upload completes, a completion message is displayed with a summary what was uploaded. + +Grafted accessions can be used like any other accession, for example, they can be used on field layouts. +If you create a list of graft accessions, use the list type 'accessions'. + +Note that you shouldn't create new grafts based on other grafts. +The scion accession and the rootstock accession have to be different, otherwise they will not be created. + +## Bulk renaming of accessions + +Accessions can be renamed in bulk using the rename accessions feature. +To rename accessions, prepare a tab delimited file with two columns: the first column should have the header "old name" and contain the accession names that need to be changed. +The second column should have the header "new name" and contain the names that the accessions in column 1 should be renamed to. + +The accession renaming feature is available from the Manage-\>Accessions page. +Click on the "Rename Accessions" button. +The first step is the upload of the file with a verification step. +The verification step checks whether all the accession names in column 1 exist in the database, and whether all the accession names given in column 2 do NOT exist in the database. +Only if both conditions are met, will the "rename" button become active, otherwise an error message is displayed listing the offending accession names. + +Optionally, the old name can be automatically added as a synonym to the renamed accession, using the checkbox on the submit form. +This option is clicked by default. +Unclick the checkbox to NOT save any old names as synonyms. + +Note that accession renaming should not be undertaken lightly. +This feature is intended for special use cases, such as where accessions are created in a nursery with a name that is different from the accession name in the downstream breeding program. + +It can also be used to rename accessions in bulk that have spelling mistakes and other issues. +Please note however, that the tool does not make any attempt to change the names of associated elements, such a plots, that may have been constructed using accession names. + +Because of the many implications of accession renaming, the feature is limited to accounts with the curator role. diff --git a/docs/r_markdown_docs/managing_barcodes.Rmd b/docs/r_markdown_docs/managing_barcodes.Rmd new file mode 100644 index 0000000000..bc55965a87 --- /dev/null +++ b/docs/r_markdown_docs/managing_barcodes.Rmd @@ -0,0 +1,76 @@ +# Managing Barcodes + +SGN databases provide tools for generating barcodes for stock identification. To access “Barcode Tools” page, clicking on “Barcodes” in the “Manage” menu. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image82.png') +``` + +“Barcode Tools” page provides four options for generating barcodes: + +- Single barcode + +- Multiple barcodes + +- Plot phenotyping barcodes + +- Trial barcodes + +To generate single barcode, clicking on “Generate Barcode” link on the “Barcode Tools” page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image274.png') +``` + +In the “Generate Barcode” section, specify the name of the barcode, size of the barcode, then clicking on “Generate Barcode” + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image245.png') +``` + +The database will generate a barcode for your stock. The barcode can be printed for your stock identification. It also appears on its corresponding stock page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image264.png') +``` + +If you have a list of stocks that you want to generate barcodes, you can use “Download Stock Barcodes” section. You have three options for entering stock names: + +1. Typing in stock names, or copy and paste from other file into the box (1) + +2. Choosing a list of stocks from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button. + +3. Uploading a “Tab-delimited Text File” with stock names. + +4. Select an optional printing format from the available formats. + +You can select printer settings that you prefer in the “Printer Settings” section. After you enter stock names and specify printer settings, clicking on “Download Barcodes” button at the bottom of the page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image248.png') +``` + + +If you have a list of plots that you want to generate phenotyping barcodes, you can use “Download Plot Phenotyping Barcodes” section. You have three options for entering plot names: + +1. Typing in plot names, or copy and paste from other file into the box (1) + +2. Choosing a list of plots from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button. + +3. Uploading a “Tab-delimited Text File” with plot names. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/plot_pheno_barcode.png') +``` + +If you have a list of trials that you want to generate barcodes, you can use “Download Trial Barcodes” section. You have three options for entering trial names: + +1. Typing in trial names, or copy and paste from other file into the box (1) + +2. Choosing a list of trial from your “Lists” (2), and transferring the list into the box (1) by clicking on “paste” button. + +3. Uploading a “Tab-delimited Text File” with trial names. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_barcode.png') +``` diff --git a/docs/r_markdown_docs/managing_breeding_programs.Rmd b/docs/r_markdown_docs/managing_breeding_programs.Rmd new file mode 100644 index 0000000000..e32321fe47 --- /dev/null +++ b/docs/r_markdown_docs/managing_breeding_programs.Rmd @@ -0,0 +1,13 @@ +# Managing Breeding Programs + +New breeding programs can be added by using “Add New Program” button on the “Manage Breeding Programs” page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/breeding_programs_screenshot.png') +``` + +Clicking on the “Add New Program” button will generate a blank form for you to fill out the name and description of the breeding program that you want to add. After completing the form, click on “Add Breeding Program” button to finish the process. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/new_program_screenshot.png') +``` diff --git a/docs/r_markdown_docs/managing_crosses.Rmd b/docs/r_markdown_docs/managing_crosses.Rmd new file mode 100644 index 0000000000..2d26c8d994 --- /dev/null +++ b/docs/r_markdown_docs/managing_crosses.Rmd @@ -0,0 +1,239 @@ +# Managing Crosses + +Information for crosses can be managed using the “Crosses" option in the Manage menu. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image148.png') +``` + +## Crossing Experiment + +Different crosses in the same trial/nursery/project are grouped via "**crossing experiment**". Crossing experiments are organized based on their breeding programs. To find a crossing experiment, you can either type the crossing experiment name in the “Search” box, or look for the crossing experiment directly in its breeding program by clicking on the “**+**” icon. In each breeding program, crossing experiments can be placed directly in the breeding program, or organized in folders. The "**Folders**" section allows you to place crossing experiments in folders, move a crossing experiment in a folder to another folder, or rearrange your folders within a breeding program. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_crosses1.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_crosses2.png') +``` + +### Add New Crossing Experiment + +To add a new crossing experiment, click on "Add Crossing Experiment" link. +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_crosses_crossingtrial.png') +``` + +Required Information: + +• “**Crossing Experiment Name**": enter a name for the crossing experiment. The crossing experiment name must not already exist in the database. + +• “**Breeding program**": select a breeding program that is available in the database. New breeding programs can be added on the "Breeding program" page, accessible from the "Manage" menu. *Breeding Program Page* + +• “**Location**”: select a location for the crossing experiment. New locations can be entered on the "**Locations**" page, accessible from the "**Manage**" menu. *Location Page* + +• “**Year”**: select a year. + +• “**Description**": enter a description for the crossing experiment. + +After filling in the information, click "**Submit**" to generate the crossing experiment. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/add_crossingtrial.png') +``` + +## Cross + +### Add New Crosses + +#### Add a cross by using the "Add New Cross" dialog {-} + +To add a single new cross, click on "Add Cross" link. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_crosses_addcross.png') +``` + +Enter cross information in the popup dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/add_cross.png') +``` + +Required Information: + +• “**Crossing experiment**": select a crossing experiment available in the database. + +• “**Location**": select a location available in the database. + +• “**Cross name**": enter a name for the cross. The cross name must not already exist in the database. + +• “**Cross type**": the options for cross types are: biparental, self, open pollinated, bulk, bulk selfed, bulk and open pollinated, double haploid, polycross, reciprocal and multicross. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/new_cross_dialog_type.png') +``` + +• The “**Female Parent”** and “**Male Parent”** field are auto-complete fields for accessions that are already in the database. The parents specified will be entered in the pedigree of the new accessions generated by this cross. + +Optional Information: + +• “**Female Plot and/or Male Plot**": In addition to the accession names, specific plots used in the cross can also be added to the database. To retrieve plot names associated with each female/male accession, enter your trial name, then click "**Search Plots**". Plot names of each parental accession in that field trial will be shown in the drop-down list, you can then select the plot used in the cross. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/new_cross_dialog2.png') +``` + +Additional crossing experimental information such as pollination date, number of flowers, number of fruits, number of seeds can be specified during adding new cross. Alternatively, this information can be updated or edited directly on the "**Cross Details**" page. + +If you know the number of accessions that are generated from the cross, they can be instantiated immediately in the database by clicking the "**Add accessions for progeny**" checkbox and specifying the number. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/add_cross_info.png') +``` + +Click "Submit" to generate the cross. + +#### Upload New Crosses {-} + +To upload new crosses from an Excel file (.xls or .xlsx), click on "Upload Crosses" link. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_crosses_upload.png') +``` + +Select a crossing experiment and a location available in the database from drop-down lists and choose a file that you want to upload, then click "**Upload File**". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/upload_crosses.png') +``` + +Please check spreadsheet format carefully. The file must be an Excel file (.xls or .xlsx). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_upload_format.png') +``` + +### Update Crosses by Uploading + +To upload progenies and/or experimental info of crosses already in the database, go to "**Manage-Upload**" page. + +In the "**Crosses**" section, there are links for uploading progenies and experimental info. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_upload_page.png') +``` + +Please check spreadsheet format in each link carefully. The file must be an Excel file (.xls or .xlsx). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/progenies_upload_spreadsheet.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/crossinfo_upload_spreadsheet.png') +``` + +Note: crossing experimental information is customized based on the need for each crop. As a result, column headers for experimental info in your database may be different from the information shown in this manual. + +## Cross Wishlist + +An Android ODK application is being developed to record cross information on a mobile device in the field. To link this mobile application with the database, the Cross Wishlist can be used to create a plan for which crosses to perform. + +This tool is available on the Manage Cross page. It is currently only available on certain databases, so when you click this link you may see an alert mentioning that the cross wishlist is not available on your database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_cross_click_cross_wishlist.png') +``` + +### Create a Cross Wishlist + +#### Step 1. Select the accessions to be crossed in your trial {-} + +There are two interfaces for this step, either "Not Using Lists" or "Using Lists". Depending on if you already have a list of female and male accessions to use, you can decide on which interface to use. The end result of using either interface is the same. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_initial_dialog.png') +``` + +We will start by showing "Not Using Lists". First select the trial in which the crosses are to be performed. This will populate a select box with all the accessions used in that trial. From here, one or many accessions can be selected as the female accession. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_not_using_list_01.png') +``` + +Once the female accessions are selected, a table is populated. Each row in this table begins with the female accession that was selected, followed by a select box with all the accessions used in the trial. From here, one or many accessions can be selected as the male to use in the cross. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_not_using_list_02.png') +``` + +Once the male accessions are selected to cross with each female accession, a table indicating priorities appears. Priority is meant to indicate an order in which to attempt the cross; first the highest priority male will be considered, but if this cross is not possible then subsequent males will be considered. An equal priority can be given and this will not indicate a specific order to follow. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_not_using_list_03.png') +``` + +Alternatively, we could have used the "Using List" interface instead. Here we select the trial in which the crosses will be performed and we provide a list of accessions to consider for the females and the males to be crossed. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_using_list_01.png') +``` + +#### Step 2. Select the female plots to be considered in the crosses {-} + +After selecting your lists, the table below is populated. The first column has all the female accessions specified and the header row has all the male accessions specified. The males to consider crossing with each female are indicated with priority. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_using_list_02.png') +``` + +After female and male accessions are selected to cross, either by the "Nor Using List" or "Using List" interface, click Next. The next dialog will allow selection of specific female plots to use for the cross. Sections for each female accession selected will appear with the field layout displayed. Selecting all plots in which the female is present indicates that the cross should be performed on all plots where that female accession is present. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_wishlist_not_using_list_04.png') +``` + +#### Step 3. Transfer the cross wishlist to your mobile crossing application {-} + +Clicking "Push Cross Wishlst for ODK Use" will send the cross wishlist plan to the ONA server for use by the mobile ODK application. Crosses can then be performed and recorded in the field using the mobile application. Afterwards, the crosses are sent back to our database and stored. + +## Crossing Experiment Detail Page + +Information for crosses in the same crossing experiment is compiled in the crossing experiment detail page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/crossingtrial_1.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/crossingtrial_2.png') +``` + +Each cross name, female parent, male parent, female plot and male plot has a link to its own detail page, which contains information specific to each one. +Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. + +## Cross Detail Page + +Information of each cross can also be viewed in its detail page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_page_1.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_page_2.png') +``` + + This page allows you to update or edit crossing experimental information and add progenies related to that cross. + Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/edit_cross_info.png') +``` + + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cross_add_progenies.png') +``` diff --git a/docs/r_markdown_docs/managing_downloads.Rmd b/docs/r_markdown_docs/managing_downloads.Rmd new file mode 100644 index 0000000000..2d2e3233a0 --- /dev/null +++ b/docs/r_markdown_docs/managing_downloads.Rmd @@ -0,0 +1,21 @@ +# Managing Downloads + +You can download phenotype, trial meta-data, pedigree, GBS genotype and GBS genotype QC files from the database to your computer by using “Lists”. To download, clicking on “Download” in the “Manage” menu. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image255.png') +``` + +For each category, you can select a list of accessions from your “Lists” to download their phenotypes, pedigree, GBS genotype, GBS genotype QC. In the case of downloading trial meta-data, you would provide a list of trials, while for downloading phenotype and GBS genotype QC, you can also use a list of trials or traits. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_meta_data_download.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image287.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image182.png') +``` diff --git a/docs/r_markdown_docs/managing_field_trials.Rmd b/docs/r_markdown_docs/managing_field_trials.Rmd new file mode 100644 index 0000000000..906bad1cf5 --- /dev/null +++ b/docs/r_markdown_docs/managing_field_trials.Rmd @@ -0,0 +1,660 @@ +# Managing Field Trials + +To view trial details on the database, click on the "Field Trials" link under the "manage" menu on the toolbar. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image290.png') +``` + +Clicking on the "Field Trials" link will bring you to the "Manage Trials" page. On this page, trials are organized according to their breeding programs. To access trial details, click on the + icon next to your breeding program. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image153.png') +``` + +Trials can be placed directly in their breeding program. Alternatively, they can be organized by using folders within each breeding program. Clicking on trial name will take you directly to the trial details page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image279.png') +``` + +## Trial Detail Page + +The trial detail page displays important information about individual trials including breeding program, location, year, description of the trial, design, and any files associated with that trial. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_start.png') +``` + +Below the trial details you will find various menus for accessing and modifying trial data. There are sections for printing labels for your plots or plants, recording phenotypes, viewing your trial layout or design, viewing phenotypes for this trial, or conducting analyses. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics(c('assets/images/trial_detail_page_navigator_1.png', 'assets/images/trial_detail_page_navigator_2.png', 'assets/images/trial_detail_page_navigator_3.png')) +``` + +The "transplanting date" field feature will only be displayed if it has a value. To add a transplanting date after creating a trial, change the show_transplanting_date parameter from 0 to 1 in the SGN config file. As a result, you will be able to add a date under the transplanting date field by clicking the "Edit Trial Details" on the trial detail page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/add_transplanting_date.png') +``` + +## Adding Trials + +Only users with the account status of "submitter" may create trials. To learn how to change your account status from "user" to "submitter" visit section \@ref(managing-your-account). + +### Prerequisites + +- To add a trial, all of your accessions should already exist in the database before you begin to design a trial. If you have accessions that are not in the database, see the instructions in [*Managing Accessions*](#managing-accessions). + +- The breeding program and location for your trial should also exist in the database. If you need to add breeding program and/or location to the database, see the instructions in [*Managing Breeding Programs*](#managing-breeding-programs) and [*Managing Locations*](#managing-locations) respectively. + +On the “Manage Trials” page, there are two methods to create trials: by selecting "Upload Existing Trial(s)" to create a trial or trials from a spreadsheet; or by selecting "Design New Trial" and entering the trial data by hand. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_manage_trials.png') +``` + +### Adding a trial by using “Design New Trial” form + +#### Step 1. Begin the "Design New Trial" workflow {-} + +Click on “Design New Trial” to begin. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_open_form.png') +``` + +The first step in this workflow is an introduction that looks like this: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_1.png') +``` + +Here it gives information about what is required for a trial, including that to create a new trial, you need to create a list of the accessions that you would like to use in the trial. Lists can be viewed, created, and modified with the "lists" tool at the upper right of the screen. For more information on lists, click [here](#working-with-lists). + +#### Step 2. Enter Trial Information {-} + +On this screen you need to enter basic information about the trial, such as breeding program and location(s). You must also select a design type, such as Complete Block Design. The design is important because it influences how your genotypes are distributed and randomized over the trial. You must first click validate before proceeding to the next step. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_2.png') +``` + +#### Step 3. Enter Design Information {-} + +On this screen you need to specify a list of accessions to use in the experiment. This list must be a valid list of accessions. You must also specify all required design information, such as number of replicates. In this case, the number of blocks must be given. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_3.png') +``` + +#### Step 4. Enter Trial Linkage Information (Optional) {-} + +This next sections allows you to associate this new trial with other field trials, crossing experiments, or genotyping plates already present in the database. This is optional, and can be completed at a later date from the trial detail page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_4.png') +``` + +#### Step 5. Enter Field Map Information (Optional) {-} + +On this screen you can specify how the row and column numbers will be generated for the plots in the trial. The row and column number represent a relative position of the plot in the field. If you are not exactly sure of how you will plant the plots in the field or you have an irregular (non-rectangular) layout, you can skip this step for now. This information can be added on the Trial Detail Page once the trial is saved in the database in order to reflect exactly how the plots were planted in the field. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_5.png') +``` + +#### Step 6. Custom Plot Naming (Optional) {-} + +On this screen it is possible to change the format in which plot names will be generated for your trial. It is recommended to skip this step and just use the format generated by the database by default. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_6.png') +``` + +#### Step 7. Review Designed Trial {-} + +On this screen you can review the trial that the database has generated. + +You will see a graphical representation of the trial. The numbers on the squares represent the plot_number of each plot and on mouse hover you can see further information about the plot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_7_top.png') +``` + +You will also see a table representation of all the plots and their information. If you want to redo the randomization, you can click the "Redo Randomization" button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_7_middle.png') +``` + +At the bottom there is a brief summary of the trial followed by two buttons. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_7_bottom.png') +``` + +#### Step 8. Add Field Management Factors to your design (Optional) {-} + +You can add Field Management Factors by clicking "Add Field Management Factor(s) to Design". Clicking this opens a dialog to specify your factor. You can name this to account for fertilizer, watering regime, inoculation, or anything else. The types of management factors available is customizable in the SGN configuration file. This is optional and can be added from the trial detail page afterwards. + + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/add_management_factor_name_dialog.png') +``` + + +Click "Continue" and a dialog will appear where you can specify plots or plants (if you added plants during trial creation) for which the factor was applied. There is a select all button as well. + + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/add_management_factor_dialog.png') +``` + +#### Step 9. Saving new trial in the database {-} + +Once you are done reviewing the trial you can click "Confirm" to save the generated trial into the database. Once the trial has saved you will see the final completion screen: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_form_8.png') +``` + +### Adding a trial from an uploaded file + +If you already have trial design layout in a spreadsheet, you can add your trial into the database by using the “Upload Existing Trial(s)” button on the *Manage Trials* page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_create_manage_trials.png') +``` + +Enter the information in the workflow to upload a trial from a spreadsheet. + +#### Step 1: {-} + +The first step is to understand what the format of the trial upload is. It is important to understand that the field layout represents plots in the experiment. Each plot has a globally unique plot_name, a sequential plot_number that is unique in the trial (but not globally unique. e.g. 101, 102, 103 for three separate plots), an accession_name representing what genotype is planted in that plot, and a block_number representing design replication. Each plot can be thought of as having a row_number and a column_number representing the relative position of the plot in a grid (e.g. the top left plot is row 1 column 1 following by row 1 column 2). Each plot can be planted with an amount of seed from a seedlot, where the seedlot_name represents the specific seed packet that was used, and num_seed_per_plot and weight_gram_seed_per_plot represent amount that were transferred from the seedlot_name to the plot_name. Treatments can be applied onto plots using additional column names in your file, where a 1 represents if the treatment was applied to the plot and an empty cell means it was not applied. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_1.png') +``` + +The following page will allow you to pick a file for upload, including uploading multiple trials at once. On this page you can also inspect the file requirements for both single trial and multi-trial uploads. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics(c('assets/images/manage_trials_upload_trial_2.png','assets/images/manage_trials_upload_trial_template.png')) +``` + + +##### Minimum File requirements {-} + +- All accession names in the file must exist in the database. See adding accessions for more information. + +- The uploaded file can be excel (.XLXS or .XLS), comma-separated values (.CSV), tab-separated values (.TSV) or semicolon-separated values (.SSV). + +- The first row (header) must contain the column names: plot_name accession_name plot_number block_number is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number + +- Only accession_name, plot_number, and block_number are required. + +Minimal Example: + +| **plot\_name** | **accession\_name** | **plot\_number** | **block\_number** | **is\_a\_control** | **rep\_number** | **range\_number** | **row\_number** | **col\_number** | **seedlot\_name** | **num\_seed\_per\_plot** | **weight\_gram\_seed\_per\_plot** | **entry\_number** | +|----------------|---------------------|------------------|-------------------|--------------------|-----------------|-------------------|-----------------|-----------------|-------------------|--------------------------|-----------------------------------|------------------| +| 2018plot1 | my_accession1 | 101 | 1 | 1 | | | | | | | | | +| 2018plot2 | my_accession2 | 201 | 2 | | | | | | | | | | +| 2018plot3 | my_accession2 | 102 | 1 | | | | | | | | | | +| 2018plot4 | my_accession1 | 202 | 2 | 1 | | | | | | | | | + +When uploading multiple trials, the requirements are the same, but with additional headers to differentiate trials in the same file: trial_name breeding_program location year transplanting_date design_type description trial_type trial_stock_type plot_width plot_length field_size planting_date harvest_date. Additionally, when uploading multiple trials, you may choose to get an email when the upload is complete (see section \@ref(email-alert-for-multiple-trial-design-upload) ) + +##### File validation {-} + +- In case of errors in the uploaded file such as missing or invalid data, a window will appear listing the specific errors in the file that must be corrected before a successful upload. + +##### Uploading a trial with Treatments {-} + + +- You can upload a trial with treatments by adding additional column(s). The column header will be the treatment e.g. fertilizer, watering regime, inoculation, etc. and the values in these columns will be either 1 or empty, indicating that the treatment was applied to the plot or not. + +#### Step 2: {-} + +Once you feel that your experiment field layout is in the right format, click on to the Next Step. You will see the following form which must be filled in completely: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_3.png') +``` + +The trial name must be globally unique in the database. Please try to follow standard naming conventions for your group. + +#### Step 3: {-} + +Go to the next page where you can link this trial to other projects. Validate the form, and then you can click "Upload Trial". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_4.png') +``` + +#### Step 4: {-} + +In the case where you have uploaded an experiment using accession_names that are not already present in the database, you will be taken to this screen. If the accession_names in your file are all already in the database, this step will be skipped. +The reason it is necessary for your accessions to be in the database before you can add a trial using them is that a single accession can be used among many trials and therefore must exist as a separate entity in the database; because of this it is also very important to be careful about adding wrongly duplicated accession_names into the database. From this screen it is possible to make a new list with the missing accession_names and then click "Add Accessions to the database" to immediately resolve the issue. Once all your accessions are in the database, click to move to the Next Step. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_5.png') +``` + +#### Step 5: {-} + +In the case where you have uploaded an experiment using seedlot_names that are not already present in the database, you will be taken to the next screen. If the seedlots in your file are all already in the database, this step will be skipped. +The reason it is necessary for your seedlots to be in the database before you can add a trial using them is that a single seedlot can be used among many trials and therefore must exist as a separate entity in the database. Once all your seedlots are in the database, click to move to the next step. + +#### Step 6: {-} + +If there are any other errors with your file, such as if the plot_names are not globally unique in the database or your plot_numbers are not unique in your trial or row_number is not an integer or any other error, you will see the errors listed in the red box. It is up to you to correct these errors in your file. Simply open up the file you selected earlier in Excel and correct the issues and then save the file. Then you can click "Submit Trial" and it will resubmit it for you. You can continue to edit your file here and submit as many times as you need until it is accepted. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_7.png') +``` + +#### Completion screen {-} + +Whether you were lucky enough to submit your trial successfully on Step 2 or if you tried many times on Step 5, once your trial has been saved in the database you will see the following screen: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_upload_trial_8.png') +``` + + +### Multi-location trials + +To add multi-location trials, simply select the multiple locations while using the 'Add Trial' form. + +This will create a separate trial for each selected location, but they will share the same design and will be grouped in a single folder. + +By default each trial design will have a fresh randomization, but if desired you may check the "Use same randomization for all locations" option. + + +### Email alert for multiple trial design upload {-} + +When uploading multiple trials from a file, you have the option to receive email notifications by clicking the "Email Alert" checkbox. By default, the system will use the email address associated with your account, but you have the option to enter a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/multiple_trial_upload_with_email.png') +``` + +### Viewing Plot Layout and Trait HeatMap + +#### Viewing plot layout + +In the "Field Layout Tools and Phenotype Heatmap" section of a Trial Detail page, the trial physical layout is displayed by default. The relative position of the plots will be displayed based on the row and column positions given to the plots during the trial creation or upload steps. The plots are color-coded based on the plot's rep and block numbers and whether or not it is used as a check. Hover the mouse over the plot to see details about a specific plot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/fieldmap_trial_layout.png') +``` + +#### Viewing plot layout for multiple trials + +If there is more than one trial grown in the same physical field, the trial layouts of all of the trials can be shown together if the trials share these properties: + +
      +
    • Each trial has the same year
      • +
    • Each trial has the same location
      • +
    • The location type of the trials' location is set to Field
      • +
    • The row and column positions of all of the plots (across the related trials) don't overlap. For example, trial #1 starts at row 1 and trial #2 starts at row 10.
      • +
    + +When these conditions are met and you check the "Select Trials in Same Field" checkbox, the plots from all of the related trials will be displayed on the same field layout. The plots will be color-coded by trial. The planting order and harvest order downloads will include the plots from all of the displayed trials in the order in which the plots occur in the field. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/fieldmap_multi_trial_layout.png') +``` + +#### Tracking plot images on fieldMap {-} + +Plot images can be seen on fieldMap if a plot is associated to any image. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/fieldmap_plot_image.png') +``` + +To view plot image(s), click on a plot, a dialog will appear. In this dialog you will see a detailed overview of the plot, including what stock(s) are contained within it and their layout in the plot, if applicable. At the bottom of the dialog, you will see options to change the plot name or accession as well as view plot images. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/fieldmap_view_plot_image.png') +``` + +On the resulting dialog, click on "View Plot Images." To see more images if a plot has more that 2 images, click on See more images... Medium size of an image can be viewed by clicking on an image. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/fieldmap_display_plot_image.png') +``` + + +#### Viewing assayed trait heatmap {-} + +The phenotype heatmap can be viewed by selecting a specific assayed trait from the selectbox drop-down. Mousing over the plots highlights the plot in green and also displays the plot's field information including the selected trait's phenotype value. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/heatmap_assayed_trait_view.png') +``` + +#### Suppressing Plot Phenotype {-} + +Clicking on a plot on the heatmap would display a dialog that has a button for suppressing a plot phenotype value for a given trait. A suppressed plot value can be excluded during trial analysis and phenotype download. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_suppress_phenotype.png') +``` + +#### Correcting spatial autocorrelation + +For trials with spatial layout information and stored phenotypes, you can check and correct for spatial autocorrelation by clicking the "Calculate Spatial Correction" button above the field map. Doing so will open the spatial corrections dialog. For an in-depth review, visit the chapter on [*data analysis tools*](#data-analysis-tools). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/heatmap_spatial_correction_button.png') +``` + +### Adding additional information in the *Trial Detail* page + +After you added a new trial to the database, you can edit trial details or add more information for that trial through the *Trial Detail* page. + + #### Uploading Physical Trial Layout {-} + +You can upload physical trial layout by clicking on the “Upload Spatial Layout” button on the *Trial Detail* page. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/image332.png') +``` + +Please check file format carefully. You can find file format information by clicking on the “Spreadsheet format” on the “Upload Spatial Layout” window. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image340.png') +``` + +Spreadsheet format: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image190.png') +``` + +#### Physical Trial Layout File requirements {-} + +- All plot names in the file must exist in the database. + +- The uploaded file should be tab delimited (txt). + +- The first row (header) must contain the column names + +Example: + +| plot\_name | row\_number | col\_number | +|------------|-------------|-------------| +| plot1 | 1 | 1 | +| plot2 | 1 | 2 | +| plot3 | 1 | 3 | + +Select the trial layout coordinates file that you want to upload for this trial, then click “OK” button to upload the file. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image79.png') +``` + +The following message is displayed after the coordinates are uploaded. + +```{r echo=FALSE, out.width='50%', fig.align='center'} +knitr::include_graphics('assets/images/image276.png') +``` + + + +#### Downloading Field Map Spreadsheet {-} + +Field map spreadsheet can be downloaded if the trial has field coordinate (row and column numbers) uploaded for it plots. +To download, click on the Download FieldMap Layout link on the Trial Heatmap section. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/field_map_download_link.png') +``` + + +#### Editing Physical Trial Layout {-} + +The “Usage Help” button contains information on how to edit physical trial layout. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/spatial_layout_usage_help.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics(c('assets/images/spatial_layout_usage_help_1.png','assets/images/spatial_layout_usage_help_2.png','assets/images/spatial_layout_usage_help_3.png')) +``` + +There are two different options for editing trial layout: + +- Replacing plot accession by clicking on the plot in the layout. + +- Replacing trial accession by using the “Edit Field Layout” button. + + +To edit a specific plot, click on that plot. Enter a new accession on the “Edit Plot Info” form, then click the “Replace Plot Accession” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics(c('assets/images/fieldmap_view_plot_image.png','assets/images/replace_plot_accession_form.png')) +``` + +To replace an accession (in every plot/plant of that accession), click on the “Edit Field Layout” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image173.png') +``` + +On the “Edit Field Layout” window, click the “Replace Accession” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image87.png') +``` + +Select any accession that you want to replace, enter your new accession, then click the “Replace Trial Accession” button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image85.png') +``` + +### Downloading the Trial Layout from the *Trial Detail* page + +Click on "Download Layout" on the *Trial Detail* page under "Experimental Design". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image348.png') +``` + +The trial layout includes all information regarding the observation units in the experiment. The observation units can be plots, plants, or subplots. The trial layout can include trial design information such as the block_number and rep_number. It can also include physical map information such as the row_number and col_number, if that information is available for the trial. +The trial layout also includes information regarding treatments that have been applied in the field. +Optionally, the layout can give information regarding accession's global performance for a list of traits. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image347.png') +``` + +### Adding Plant Entries To Your Trial + +After you added a new trial to the database you can choose to add plant entries to your trial. Adding plant entries enables plant level phenotyping. It is generally better to enter data at the plant level into the database because it is always possible to calculate plot level phenotypes from the individual plant data. + +Plant entries can be added to your trial in four ways: +(1) Automatically generated by the database. The only input required is the number of plants per plot. +(2) Uploaded in an XLS or XLSX file. This allows you to specifically name your plant entries. +(3) Uploaded using plant index number instead of name (the name is automatically generated) +(4) Uploaded using the number of plants per plot. This allows different numbers of plant per plot, as opposed to option 1. + +Additionally, if a trial has had subplots added, each of these options can be used to add plants to subplots. Note that once subplots are added, plants cannot be added directly to plots, and once plants have been added, subplots cannot be created. + +These options are available in the "Plant Entries" section on the *Trial Detail* page under "Experimental Design," as shown below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_add_plant_entries.png') +``` + +#### Automatically Generate Plant Entries {-} + +Clicking on "Add plant entries" opens the following dialog box. The only input required is the number of plants per plot. This will create plant entries that are named as a concatenation of the plot_name and the plant's index number e.g. plot_name_plant_1. You may optionally add row and column data to give plants a spatial layout within each plot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_add_plant_entries_autogenerated.png') +``` + +#### Upload Plant Entries {-} + +Alternatively, you can choose to upload an XLS or XLSX file that contains the names of the plant entries, the plant index numbers, or the number of plants per plot. Each option comes with a dialog that specifies the file formats. + +### Adding Tissue Sample Entries To Your Trial + +Some trials require tissue samples to be collected from plants in a field trial. The database will generate these tissue sample identifiers for you and will maintain all relationships with the plant, plot, accession, etc. To begin, go to the Design section of a trial's detail page and open the "tissue sample entries" section. +Please note that tissue samples are directly related to plants, therefore your trial requires plants before you can add tissue samples. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_tissue_sample_default.png') +``` + +When you click on "Add tissue sample entries" you will see a dialog where you specify the number of tissue samples you require per plant. Once you have specified how many tissues samples, you can give specific words to distinguish samples, such as "root" or "stem", as seen below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_tissue_sample_create.png') +``` + +Once you have added tissue sample entries they will appear in the design section of the trial as seen below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_tissue_samples.png') +``` + +Each tissue sample has a detail page where you can add information about the sample, such as if it is in transit or in storage somewhere. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_tissue_sample_detail.png') +``` + +The related stocks section near the bottom of this detail page displays the relationships between all stocks, including tissue samples. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_trials_tissue_sample_related_stock.png') +``` + +### Uploading GPS Coordinates For Plots + +You can upload GPS coordinates for the plots in your trial. There is a link on the Trial Detail Page as shown below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_add_plot_gps.png') +``` + +Clicking on this link will bring up the following dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_add_plot_gps_dialog.png') +``` + +Here you can upload an XLS or XLSX file. To see information on the format of the file that should be uploaded, click on "Spreadsheet format". This will bring up the following dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_add_plot_gps_upload_info.png') +``` + +This dialog tells you that the file must be XLS or XLSX and must contain: plot_name WGS84_bottom_left_x WGS84_bottom_left_y WGS84_bottom_right_x WGS84_bottom_right_y WGS84_top_right_x WGS84_top_right_y WGS84_top_left_x WGS84_top_left_y +The GPS coordinates should be WGS84 format and specify a four-pointed polygon around the plot. + +### Repetitive Measurements Section + +If a trial includes repetitive traits or time-series values, you can effectively view and analyze these values through the Repetitive Measurements Section. Start by selecting the desired trait from the trait drop-down menu. Next, define the date range by either using the date-range picker or an interactive slider, which allows you to dynamically adjust the period you wish to examine. Once the date range is set, determine how to handle the repetitive measurements by choosing from various options such as First Value, Last Value, Averaged Value, Sum Values, or All Values. Choosing the "All Values" option enables an additional feature that visualizes the trend of the values over time, helping you identify patterns and trends within the data. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_view_repetitive_measurements.png') +``` + +### Uploading Additional Files To Trial + +It may be of interest to you to upload additional documents, images, or recordings to your trial. To do this, scroll down to the "Uploaded Additional File" section on the trial detail page. From here you can view and download any of these additional files. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_upload_additional_file.png') +``` + +To upload an additional file, click on the "Upload Additional Files" link. A dialog will appear where you simply select your desired file. For information, you can click "Upload information" to see the following message. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_upload_additional_file_info.png') +``` + +## Updating Trial Data + +To updated the trial-level metadata (such as the planting date, design type, description, etc) of one or more existing trials, click the "Update Existing Trial(s)" button from the Manage > Field Trials page. This upload can also be used to rename trials or move trials to a different breeding program. In order to update a trial, you must be a curator or a submitter (that is associated with the breeding program of the trials). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/update_trial_metadata.png') +``` + +Here you can upload a file that contains the new metadata for the existing trials in the database. The first column is labeled 'trial_name' and includes the name of the existing trial. Additional columns can be included for the metadata you want to update. Any columns not included in the file or values left blank will leave the existing metadata unchanged. The columns that can be included are: + +- new_trial_name: A new name for the trial, must not already exist in the database +- breeding_program: The name of breeding program that managed the trial, must exist in the database. +- location: The name or abbreviation of the location where the trial was held, must exist in the database. +- year: The year the trial was held. +- transplanting_date: The transplanting_date of the trial was conducted. Date in YYYY-MM-DD format or 'remove' to remove the date +- planting_date: Date of Planting in YYYY-MM-DD format or 'remove' to remove the date +- harvest_date: Date of Harvest in YYYY-MM-DD format or 'remove' to remove the date +- design_type: The shorthand for the design type, must exist in the database. Possible values include CRD: Completely Randomized Design, RCBD: Randomized Complete Block Design, RRC: Resolvable Row-Column, DRRC: Doubly-Resolvable Row-Column, ARC: Augmented Row-Column, Alpha: Alpha Lattice Design, Lattice: Lattice Design, Augmented: Augmented Design, MAD: Modified Augmented Design, greenhouse: undesigned Nursery/Greenhouse, splitplot: Split Plot, p-rep: Partially Replicated, Westcott: Westcott Design +- description: Additional text with any other relevant information about the trial. +- trial_type: The name of the trial type, must exist in the database. Possible values include Seedling Nursery, phenotyping_trial, Advanced Yield Trial, Preliminary Yield Trial, Uniform Yield Trial, Variety Release Trial, Clonal Evaluation, genetic_gain_trial, storage_trial, heterosis_trial, health_status_trial, grafting_trial, Screen House, Seed Multiplication, crossing_block_trial, Specialty Trial +- plot_width: plot width in meters +- plot_length: plot length in meters +- field_size: field size in hectares + +## Deleting Trial Data + +To delete a trial data, click on the "Delete trial data" section. There are links to delete traits, layout and trial entry data. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_delete_trait1.png') +``` + +To delete assayed trait data, click on "Delete trait data" link. On the appeared dialog, confirm deletion by clicking on the "Select Traits For Deletion" button, then select one or more traits to delete from the trial. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_delete_trait2.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_detail_page_delete_trait3.png') +``` + +To delete trial layout data, click on the "Delete layout data" link. Confirm deletion on the appeared dialog. + +To Delete trial entry, click on "Delete trial entry" link. Confirm deletion on the appeared dialog. diff --git a/docs/r_markdown_docs/managing_genotyping_plates.Rmd b/docs/r_markdown_docs/managing_genotyping_plates.Rmd new file mode 100644 index 0000000000..c9e5b9b1e8 --- /dev/null +++ b/docs/r_markdown_docs/managing_genotyping_plates.Rmd @@ -0,0 +1,62 @@ +# Managing Genotyping Plates + +Genotyping Plates represent the content of a genotyping plate sent to a genotyping facility (e.g. samples in specific wells). To streamline this process, it is possible to upload this information or let the database create a plate for you. +Once the genotyping plate is saved in the database it is then possible to export the information directly to genotyping facilities that are BrAPI compliant. The genotyping facility can then provide status information to us via BrAPI. + +To begin go to Manage->Genotyping Plates. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials.png') +``` + +Here the genotyping plates are divided by Breeding Program. These sections can be expanded by clicking on one. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_expand.png') +``` + +## Adding a New Genotyping Plate + +To begin, click on "Add Genotyping Plate". Notice that this form is split into three sections: "Plate Information", "Well Information", and "Confirm". The first section is for defining information about the genotyping plate, such as a Plate identifier, plate format (96 well), etc. The second section is for defining the samples in the wells, such as sample names, sample concentrations, well position, etc. The final section is for Submitting the info. + +All fields in the Plate Information section are required. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_add_trial.png') +``` + +In the Well Information section you can choose to either 1) Upload an XLS or XLSX spreadsheet with your sample layout or 2) let the database create the sample layout. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_add_trial_wells.png') +``` + +If you choose to upload an XLS or XLSX spreadsheet, the Spreadsheet Template info requires the following: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_upload_template.png') +``` + +In either case, the sample identifier is generally a concatenation of Plate name and well position, e.g. MyGenotypingTrial1_A01. +In either case, you need to provide a "source_observation_unit_name" for each sample. This can be a tissue sample name, a plant name, a plot name, or an accession name; however, in any case, the identifier must already exist in the database. This allows us to link the sample in the well to specific field trial plots, or, plants, or tissue_samples. If you only know which accession is in the well, you can use the accession name. + +In the final Confirm section you can decide whether to submit this information to the genotyping facility you selected. This requires that the genotyping facility is BrAPI compliant to work. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_add_trial_confirm.png') +``` + +## Genotyping Plate Detail Page + +If you open a specific genotyping plate, it will take you to the detail page. +Here you can see the Accessions used in the plate (if you created the trial and the source_observation_unit_names you used were plots, this will still work because we know the accession of the plot or plant or tissue sample). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_detail_info.png') +``` + +Further down you can see a graphical representation of your plate with well positions. This can be 96 well or 384 well depending on your plate format. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_trials_detail_table.png') +``` diff --git a/docs/r_markdown_docs/managing_high_dim_phenotyping_data.Rmd b/docs/r_markdown_docs/managing_high_dim_phenotyping_data.Rmd new file mode 100644 index 0000000000..c72dbf5ce1 --- /dev/null +++ b/docs/r_markdown_docs/managing_high_dim_phenotyping_data.Rmd @@ -0,0 +1,51 @@ +# Managing High Dimensional Phenotyping Data + +## Managing Transcriptomic Data + +### Uploading Transcriptomic Data + +To upload transcriptomic data, go to the transcriptomics page by clicking "Transcriptomics" under the Manage tab. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/transcriptomics_menu.png') +``` + +Click the "Upload Transcriptomics Data" button to open the upload workflow dialog. On the second step "Samples", you will be prompted to make sure that your samples are already in the database. You can refer to the "Managing Tissue Samples" chapter for instructions on how to create a sampling trial. After creating a sampling trial or confirming that your samples exist, move on to the "Protocol Info" step. Here you can select an existing protocol, or create a new protocol. To create a protocol, click the "Protocol Not Shown. Create a New Protocol" button. From here you can fill in information about your new protocol and when finished, click "Go to Next Step". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/create_transcriptomics_protocol.png') +``` + +You will need to have two csv files when uploading transcriptomics data. The first is the data matrix file, which should have these headers: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/data_matrix_headers.png') +``` + +The names in the sample_name column should match the sample names you created in your sampling trial file. The header marked "transcript_name_columns" should be replaced with the transcript names you want to upload, each in its own column in the header and the respective expression value below. Example: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/data_matrix_example.png') +``` + +The second file is the transcript details file, which should have these headers: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/transcript_details_headers.png') +``` + +Each row under the transcript_name column should contain each transcript name from your data matrix file. For example: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/transcript_details_example.png') +``` + +Once these two files are uploaded, click verify to check that they are formatted correctly and that the samples exist. If there are no errors, you can then click store to store your transcriptomics data. + +### Downloading Transcriptomics Data + +To download your data, click the "Uploaded Transcriptomics Data" dropdown menu on the Transcriptomics page. Then click the "Your Uploaded Transcriptomic Data" dropdown. From here you will be able to see the transcriptomics data that you have uploaded. Click the "Download" button next to the data that you wish to download. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/download_transcriptomics.png') +``` diff --git a/docs/r_markdown_docs/managing_image_data.Rmd b/docs/r_markdown_docs/managing_image_data.Rmd new file mode 100644 index 0000000000..257f745d4a --- /dev/null +++ b/docs/r_markdown_docs/managing_image_data.Rmd @@ -0,0 +1,132 @@ +# Managing Image Data + +## Image-Phenotyping Dashboard + +1. Upload raw image-captures in a compressed file (.zip) for orthophotomosaic assembly or upload previously stitched orthophotomosaic raster (.PNG, .JPG) imagery. +2. Dashboard shows all field trials and uploaded imaging events in collapsible sections. +3. Follow standard processes to manually create templates for assignment of plot-polygon images to the field experiment design. +4. All imagery is shown with the spectral category within collapsible sections. Figure shows NIR imagery. +5. Apply Fourier transform filtering, thresholding, and vegetation index masking. Plot-polygon images for all image processes are shown. +6. Extract and export phenotypic values from plot-polygon images for analyses and model training. + +## Image Input + +Clicking "Upload Imagery" will open the following dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_intro.png') +``` + +Raw-captures can be uploaded in a compressed (.zip) file so that they can be assembled into an orthophotomosaic. If orthophotomosaic assembly is not required, raster images (.PNG, .JPG) can be uploaded. Example data is given for raw Micasense RedEdge 5-band multispectral captures and for stitched orthophotomosaics. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_field_trial.png') +``` + +To begin uploading images, a field trial must be selected. The field trial must already be saved in the database. For information about adding a field trial, please read the Field Trial documentation. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_drone_run.png') +``` + +The image data is added to an imaging (drone run) event. Here you can select a previously saved imaging event or you can create a new one by defining a name, description, and date. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_image_info_1.png') +``` + +The uploaded data can be raw image-captures or complete raster images. Here you can select whether orthophotomosaic stitching is required. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_image_info_2.png') +``` + +In the case that orthophotomosaic stitching is required, select 'yes'. On the next step you will see the following: +Upload a zipfile with the raw-captures. +When uploading Micasense RedEdge raw-captures, provide images of the Micasense calibration panels in a zipfile as well. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_images_zipfile.png') +``` + +In the case that orthophotomosaic assembly is not required, simple upload the raster images. Select the number of image bands that will be uploaded e.g. for a five band multispectral camera, select 5. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_image_info_3.png') +``` + +In the caes that orthophotomosaic stitching is not required, select 'no'. On the next step you will see the following: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_upload_images_rasters.png') +``` + +Upload an image at each band with a unique name, description, and spectral type. + +## Standard Process + +Once imagery is uploaded, it will appear on the dashboard under the field trial. Clicking the "Run Standard Process" button will begin extracting plot-polygon phenotypes from the imagery. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_button.png') +``` + +Clicking the button will open the following dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_intro.png') +``` + +Select a drone run band to use in this process. In the case of the Micasense 5 band multispectral camera there will be 5 bands shown here; select the NIR channel in this case because it has the highest contrast. In the case of standard color images, there will only be the RGB Color Image option here. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_drone_run_band.png') +``` + +Rotate the image so that there the plots are oriented in a grid fashion. There can be a skew in the field layout, as seen in the following example. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_rotate.png') +``` + +Perform a rough cropping of the image by clicking on the four corners of the field. Cropping is important to remove any extraneous parts of the image. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_cropping.png') +``` + +This step shows a histogram of the cropped image. The standard process will magnitude threshold the top and low ends of the distribution. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_threshold_hist.png') +``` + +In this step, the template for the plot polygons in the experimental field design are associated to the image. First, defined the number of rows and columns in the field experiment. Then click the four corners of the image, in respect to the top right, top left, botton left, and bottom right positions. Next click on "Draw Plot Polygon Template". Review the template and clear/repeat the process until the template matches well. It is possible to "copy/paste" templates in the case where there are large breaks in the field design. Next, scroll down to the "assign Plot Polygons to Field Trial Entities" section. Select the location of Plot Number 1 as either "top left" or "top right" and whether the field design is serpentine or zigzag. Click on "Generate Assignments" and review that the names of the plots appear correctly in the overlay on the image. Finally, click "Finish and Save Polygons to Plots" when you have have confirmed the assignments. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_plot_polygon.png') +``` + +Next, the dialog shows you that the standard process will be repeated for all uploaded image bands. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_apply.png') +``` + +Next, choose which vegetation indices to apply. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_vegetation_index.png') +``` + +Next, choose the phenotypic values to extract. You must define the time point for which the phenotype is; if the field trial has a planting date, the time point will automatically be populated as image date minus the planting date. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_image_phenotyping_standard_process_phenotypes.png') +``` + +After completing the standard process, the job will continue in the background until it completes. You can check the status of the job from the dashboard. + +## Ground Control Points + +Ground control points can be saved after an imaging event has undergone the standard process on orhomosaics. Ground control points can then be used across imaging events on the same field experiment in order to automate the entire standard process. diff --git a/docs/r_markdown_docs/managing_locations.Rmd b/docs/r_markdown_docs/managing_locations.Rmd new file mode 100644 index 0000000000..5845b38e56 --- /dev/null +++ b/docs/r_markdown_docs/managing_locations.Rmd @@ -0,0 +1,19 @@ +# Managing Locations + +Field locations can be managed using the “**Manage Locations**” page. On this page, locations in the database are organized based on their breeding programs. Each location has a link to trials conducted in that location. To add a new location, click on the “Upload New Locations” button that links to the “Upload Locations” form. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image141.png') +``` + +The "Upload Locations" describes how to build a spreadsheet with location data for upload. Name, abbreviation, country code, country name, program, type, latitute, longitude, and elevation are all required. The NOAA station ID is optional. Link a spreadhsheet to the form and click "Upload" to add those locations to the database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image187.png') +``` + +Alternatively, locations can be viewed and added via the map. Hover over an icon on the map to see the location details and trials linked to that location. Click on the map to open the new location dialog. Fill in the same information that would be used in the spreadsheet upload to add a new location. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/location_map.png') +``` diff --git a/docs/r_markdown_docs/managing_observation_variables.Rmd b/docs/r_markdown_docs/managing_observation_variables.Rmd new file mode 100644 index 0000000000..bae42e8c99 --- /dev/null +++ b/docs/r_markdown_docs/managing_observation_variables.Rmd @@ -0,0 +1,54 @@ +# Managing Observation Variables + +## Managing Observation Variables with Traits, Methods, and Scales + +Observation variables are the identifiers used when collecting phenotypic data. An observation variable is composed of a trait, a method, and a scale. The trait describes the attribute being measured e.g. 'Plant Height'. The method defines the protocol in which the trait was observed e.g. 'Using a one meter long measuring stick'. The scale defines the units or dimensions for which the measurement was taken e.g. 'Meters'. + +Generally, observation variables are defined in ontologies that are predefined. We often use ontologies from cropontology.org. In this case, you will not be able to define your own observation variables directly; instead, you will need to contact us and we will add the observation variable for you. + +For databases where the user has greater control, we have an interface to allow addition of observation variables, along with traits, methods, and scales. To begin, go to the Search->Traits page. + +If the database you are on allows you to directly add observation variables, you will see the following button at the bottom of the page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_start_button.png') +``` + +When you click the button, the following workflow will appear. You should be logged in or else it will not allow addition of the observation variable. +The workflow begins with an introduction. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_intro.png') +``` + +On the next workflow step, you select the ontology that you want to insert the new observation variable into. You must also give a name and a definition for the new observation variable. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_observation_variable.png') +``` + +On the next workflow step, you select the trait ontology to use. Once you select a trait ontology, a select containing all the terms in the selected ontology will appear. You can either select a trait or if it does not exist in the select, you can create a new one by giving a name and a definition for the new trait. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_trait.png') +``` + +On the next workflow step, you select the method ontology to use. Once you select a method ontology, a select containing all the terms in the selected ontology will appear. You can either select a method or if it does not exist in the select, you can create a new one by giving a name and a definition for the new method. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_method.png') +``` + +On the next workflow step, you select the scale ontology to use. Once you select a scale ontology, a select containing all the terms in the selected ontology will appear. You can either select a scale or if it does not exist in the select, you can create a new one by giving a name and a definition for the new scale. You can also define a format, minimum, maximum, categories, and default value for the new scale. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_scale.png') +``` + +On the last page of the workflow, you confirm the submission. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_observation_variables_workflow_submit.png') +``` + +Afterwards, you can use the newly created observation variable ontology term in your phenotyping. diff --git a/docs/r_markdown_docs/managing_odk_collection.Rmd b/docs/r_markdown_docs/managing_odk_collection.Rmd new file mode 100644 index 0000000000..1773d8d0b4 --- /dev/null +++ b/docs/r_markdown_docs/managing_odk_collection.Rmd @@ -0,0 +1,44 @@ +# Managing ODK Data Collection + +To access this page go to Manage and then ODK Data Collection. +ODK is used for remotely collecting data on Android and IOS devices. We currently are working to support two ODK service providers, namely ONA and SMAP. We are using ONA to collect crossing information, including all lab activities following seed production. We are using SMAP for phenotypic data collection. + +## ONA Crossing Information + +### Managing ONA Crossing Information + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/odk_ona_management.png') +``` + +To begin collecting data using the ONA ODK form you must first have a crossing plan in the form of a Cross Wishlist. To do this from this page, click the "Export Cross Wishlist to ONA" button. Please refer to the "Create Cross Wihlist" help section for more information. +It is possible to view the current available cross wishlists by clicking the "Export Cross Wishlist to ONA" button and then clicking "Available Cross Wishlists". + +Once your cross wishlist is available, you can use your mobile ODK application to record crosses being done realtime. You can also record all laboratory activities following seed extraction up to greenhouse plantlet hardening. + +As you collect data using your mobile ODK application, your responses will be synchronized with our database. The "Schedule Import for Selected Form" section gives you options to perform the import daily or more frequently. It is also possible to initiate a data import from ONA at anytime by clicking "Import Crossing Data from Selected Form on ONA". + +### Reviewing Plant Status + +The mobile ODK application has options to collect information about the status of plants in the field, such as if they are flowering. Images for each plant can also be recorded. +The database will report this information here in a summary table that looks like the following. Notice that images are also transferred to the database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/odk_ona_cross_status.png') +``` + +### Graphical Summary For Performed Crosses + +There is a section to summarize activities done for each cross. In this table each row represents a single cross performed. All the activities that have been performed will be shown here, such as "first pollination" and "embryo rescue". The scatter plot shown tracks seed numbers generated on the Y axis and date of activity on the X axis. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/odk_ona_cross_progress_graph.png') +``` + +### Summary Information For Performed Crosses + +There is a secondary section to summarize what has been done across the entire Cross Wishlist. This tree structure shows all activities performed for a cross and shows how these crosses relate to the Cross Wishlist. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/odk_ona_cross_progress_tree.png') +``` diff --git a/docs/r_markdown_docs/managing_outliers_in_dataset.Rmd b/docs/r_markdown_docs/managing_outliers_in_dataset.Rmd new file mode 100644 index 0000000000..e09ed995da --- /dev/null +++ b/docs/r_markdown_docs/managing_outliers_in_dataset.Rmd @@ -0,0 +1,64 @@ +# Managing Outliers in Dataset + +## What is Outliers Functionality in Dataset ? + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/outliers_dataset_icon.png') +``` + +As in step [The Search Wizard] +we can create a dataset. + +The dataset incorporates a feature to identify outlier points, which we may choose to exclude from a specific dataset. It's important to note that these exclusions only apply at the dataset level, and no data is permanently removed from the database. Additionally, outlier categorization can be modified at any time, and these changes are visible to all other functionalities within the system. + +Each dataset stores a wholly unique set of outlier points, completely independent of any other dataset in the database. Outliers are specifically designated for traits within datasets, exclusively encompassing phenotype data. If a particular dataset lacks traits as a part of wizard selection, this functionality is not available. + +Each trait has its own set of defined outliers. + +## Accessing Trait Visualization + +Once you've selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/outliers_dataset_basic_panel.png') +``` + +## Interpreting Visual Elements + + +Once you've selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. + +- **Green Points**: As per the legend, represent values for the selected trait that fall below the cut-off point set by the slider. (non-outliers) +- **Black Outlined Points**: These data points are outlined with black borders, indicating that they are currently designated as outliers in the database. +- **Red Points**: The red data points denote the cut-off points established by the slider for the allowable deviation value. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/outliers_dataset_full_visualisation.png') +``` + +## Choosing Cut-Off Values + +You have two fundamental options for setting cut-off points: + +- **Median with MAD**: This option involves using the median (middle value) along with the Mean Absolute Deviation (MAD) as a reference point for determining cut-off values. +- **Mean with Standard Deviation**: Alternatively, you can choose to use the mean (average) in conjunction with the Standard Deviation to set cut-off points. + +## Setting Deviation Multiplier + +The slider allows you to specify the deviation multiplier from a central point, which influences the cut-off values. + + +## Utilizing Graph Controls + +Beneath the graph, you'll find four buttons, each serving a distinct function: + +- **Add selection to outliers**: This button enables you to save the current cut-off points to the database for future reference. +- **Reset outliers for current trait**: You can use this option to reset outliers for the selected trait. +- **Reset all outliers**: This button allows you to reset outliers for the entire dataset. +- **Download Phenotype Table without outliers**: You can download the phenotype data table in a comma-separated value format file, using this feature, with outliers excluded for selected dataset. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/outliers_dataset_actions.png') +``` + +These tools and functions are designed to provide you with control and insights when working with data visualization and outliers. \ No newline at end of file diff --git a/docs/r_markdown_docs/managing_phenotypic_data.Rmd b/docs/r_markdown_docs/managing_phenotypic_data.Rmd new file mode 100644 index 0000000000..1f7d3f5033 --- /dev/null +++ b/docs/r_markdown_docs/managing_phenotypic_data.Rmd @@ -0,0 +1,63 @@ +# Managing Phenotypic Data + +To facilitate uploading process for phenotypic data, “Manage Phenotypic Data” page provides two options for uploading: Field Book Phenotype file in database format and phenotype file in Excel (.xls or .xlsx) file format. To access “Manage Phenotypic Data” page, clicking on “Phenotyping” in the “Manage” menu. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image178.png') +``` + +## Uploading Fieldbook Phenotypes + +### Export Field Book Database File + +The database upload of Field Book phenotype data relies on the "Database" format from the Field Book. Please make sure to export the "Database" format from the Field Book if you intend to upload the data using the Field Book Upload we describe below. If you prefer to use the "Table" format that the Field Book exports, you can modify this format to work with the Speadsheet Upload we describe below. + +### Upload Field Book Database File + +To upload a Field Book Phenotype file in a database format, click the “Upload Fieldbook” link + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image185.png') +``` + +The “Upload Fieldbook” link on this page and “Upload” link on the “Field Book Tools” page open the same dialogue. Please follow instructions for uploading phenotypic files on the \@ref(using-fieldbook-app) page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image112.png') +``` + +## Uploading Spreadsheet Phenotypes + +To upload a phenotype file in an Excel (.xls or .xlsx) file format, click the “Upload Spreadsheet” link. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image288.png') +``` + +Please specify “Data Level” (Plots or Plants) and select the Excel file that you want to upload. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image68.png') +``` + +### Generating Spreadsheet File + +You can find more file format information by clicking on “Spreadsheet Format” link. Clicking on "Spreadsheet Format" will open the following dialog. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/phenotype_spreadsheet_upload_help_dialog.png') +``` + +Clicking on "Create Phenotyping Spreadsheet" will bring up a dialog where you can indicate the trial(s) you are interested in and the trait list you are interested in. Clicking "Submit" will download the xlsx file onto your computer, where you can then fill in the phenotypes. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/create_phenotype_spreadsheet_dialog.png') +``` + +### Uploading Spreadsheet File + +To ensure that the file has a correct format for uploading, click on the “Verify” button. This will check the contents of the file and also perform quality checks on the values in the file. These checks include checking the trait definition for categorical values, minimum and maximum values, and data type checking. It will also check if there are already values uploaded for the given observation units and traits. If there are, there is an option to overwrite the existing values with the new values in your file. If the file is valid, only then can you click "Store" to store the information in the database. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/phenotype_spreadsheet_upload_dialog.png') +``` diff --git a/docs/r_markdown_docs/managing_phenotyping_image_data.Rmd b/docs/r_markdown_docs/managing_phenotyping_image_data.Rmd new file mode 100644 index 0000000000..28de40b15f --- /dev/null +++ b/docs/r_markdown_docs/managing_phenotyping_image_data.Rmd @@ -0,0 +1,35 @@ +# Managing Phenotypic Image Data + +## Uploading Image files + +Clicking "Phenotyping" under Manage, and then "Upload Images" will open this dialog: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image_upload_dialog.png') +``` + +If uploading images directly, select the file format "Images". A single image or multiple selected images can be uploaded using this file format. Each of these images must have been downloaded from the Fieldbook app, or have a filename that follows this structure: observationUnitName, traitname, number, timestamp joined by underscores. The filename can not include any underscores other than the ones separating each value. For example: 38873_branching_5_2025-07-28.jpg + +If the images have been taken in the Fieldbook app, they will have EXIF data that includes their associated stocks and traits. This EXIF data will be automatically parsed and the associations will be made when the images are uploaded. + +After selecting your images for upload, clicking verify will check if your filenames or image metadata is correct and corresponds to existing stocks in the database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/filename_verify_success.png') +``` + +If the verification is successful, click store to save these images in the database. + +If uploading images with associated phenotypic data, select the file format "Images with Associated Phenotypes". A spreadsheet with the phenotypic values needs to be uploaded alongside a zipfile of images. The spreadsheet can be in .xls or .xlsx format, and the column headers need to be: + +observationUnitName \| observationVariableName \| value \| timestamp \| image_name \| person + +The image_name column should match the file names of the associated images in the zipfile. + +## Uploading Images Directly to Stocks + +In the Images section on a stock details page you can add a new image associated with that stock by clicking the "Add Image" button. This will associate any images uploaded through this dialog with the corresponding stock. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/stock_add_image.png') +``` diff --git a/docs/r_markdown_docs/managing_populations.Rmd b/docs/r_markdown_docs/managing_populations.Rmd new file mode 100644 index 0000000000..fba48e541f --- /dev/null +++ b/docs/r_markdown_docs/managing_populations.Rmd @@ -0,0 +1,25 @@ +# Managing Populations + +Populations are modeled as groups of accessions. This grouping can be useful in downstream analyses. To manage these populations go to Manage Accessions and scroll tp the bottom. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_populations.png') +``` + +To add a new population click "Create Population". The following dialog will appear where you choose a list of accessions and give a name to the new population. Please note it is also possible to create a population when you are uploading new accessions into the database. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/manage_populations_add_population.png') +``` + +Click on the plus (+) button next to Populations to see all the available populations. Click on a population name to see the accessions in the population. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_populations_expand_table.png') +``` + +From here you can delete accessions from a population as well as add new accessions to the population. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_populations_table.png') +``` diff --git a/docs/r_markdown_docs/managing_seedlots.Rmd b/docs/r_markdown_docs/managing_seedlots.Rmd new file mode 100644 index 0000000000..52420c54d7 --- /dev/null +++ b/docs/r_markdown_docs/managing_seedlots.Rmd @@ -0,0 +1,185 @@ +# Managing Seed Lots + +Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. The seed in seedlots can be from crosses or can be named accessions. Seedlots from crosses would represent seed harvested. Click Manage and then Seed Lots to begin. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlots.png') +``` + +## Add New Seedlot(s) + +To add a single new seedlot, click on "Add Seedlot". This will bring up the following dialog where you enter information about where the seedlot exists, what accession or cross is contained in it, and how many seeds there are. A seedlot must contain either an accession or a cross, and not both. A seedlot must have a weight in grams or a seed count or both of these. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_add_seedlot.png') +``` + +In the case where you have many seedlots to add to the database, you can upload an excel XLS or XLSX file instead. Click "Upload Seedlots" to see the following dialog. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_upload_seedlots.png') +``` + +## Seedlot Transactions + +Seedlots are capable of tracking where seeds came from, such as from crosses, and to where seeds go, such as to plots in the field. If you navigate to a seedlot detail page you will see the following. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_detail_page.png') +``` + +On this page you see and can edit information regarding a single seedlot, such as its name and location. You will also see a table indicating all t he transactions that a seedlot has been involved in, such as if it was planted in a plot in the field. Transactions to field plots are created when adding or uploading a new trial or from a trial's detail page. Clicking on "Add New Transaction" let you add a transaction from between this seedlot and another seedlot. This kind of transaction is useful for representing if you have distributed seed to different locations. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_add_new_transaction.png') +``` + +## Seed Inventory + +To inventory your seed: +1) Make sure your seedlots are in the database. Use "Add New Seedlot" to add a single seedlot or "Upload New Seedlots" to add many. +2) Make sure your seedlots are barcoded. You can print these barcodes from the database. +3) Use the "Inventory" Android Application to scan seedlot barcodes and record weight. Then use "Upload Inventory" to upload this info into database. If you prefer you can create your own CSV file and upload that, if you do not want to use the Inventory Application. +For more info about the "Inventory" Android Application go to Inventory. + +Clicking the "Upload Inventory" button will bring the following dialog: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_upload_inventory.png') +``` + +The CSV file that should contain your inventory should meet these Template requirements. The Seed Inventory Android Application exports this exact file. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_upload_inventory_template.png') +``` + +## Find Seedlots For a List of Accessions + +A convenient tool for searching available seedlots for a list of accessions is available in the list tool. First open up your list of accessions. For help opening a list of accessions please see the List section help. There is a button called "See Available Seedlots". + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_accession_list_search.png') +``` + +Once you click this, you will see the following table in a dialog. From here you can create a list of seedlots using the checkboxes and the input at the bottom. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_list_available_seedlots.png') +``` + +## Create a seedlot for an Accession or Cross + +Complementary to what we saw above for creating seedlots from the "Manage Seedlots" page, it is possible to create a new seedlot from an accession's detail page or from the cross detail page. On the accession detail page, this is visible in the "Related Stocks" section as seen below. The cross detail page has an identical section. Notice the link for creating a new seedlot, which streamlines adding the seedlot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlot_accession_detail.png') +``` + +## Add quality data to a seedlot + +Quality information can be added to a seedlot in the quality field. This is also available as a column in the file upload format. It is recommended to use a controlled vocabulary, defined by the user, for the quality field. For example, good quality seed should be labelled "ok", whereas other quality descriptors could be "moldy", "insect damage", or "low sprouting", etc. + +## Seedlot Maintenance Events + +For some crops, such as sugar kelp, a "seedlot" requires routine maintenance for the successful long-term storage of the seedlot. (For example, a Seedlot Maintenance Event for sugar kelp would be the routine change of the water that gametophytes are kept it). Breedbase can now store a record of these Seedlot Maintenance Events associated directly with existing Seedlots. Maintenance Events can be uploaded using a simple Excel template or recorded directly on the website. + +### Setup + +Each Breedbase instance needs to be configured to support the storage of Seedlot Maintenance Events since each crop will have their own distinct set of maintenance events for their seedlots. To check if your Breedbase instance supports this feature, go to the Manage menu and select the Seed Lots page. Make sure you are logged in and look for the **Seedlot Maintenance** button near the top, next to the **Create Seedlot(s)** and **Upload Inventory** buttons. If you don't see this button, contact the developer(s) supporting your Breedbase instance and ask if they can setup this feature. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlots_seedlot_maintenance.png') +``` +*The location of the Seedlot Maintenance button on the Manage > Seed Lots page* + +### Adding Events + +Seedlot Maintenance Events can be added using two methods: 1) Uploading an Excel template or 2) Recording events directly on the website + +#### Uploading Events with Excel Template {-} + +To bulk-upload a file of Seedlot Maintenance Events, first create an Excel (.xls or .xlsx) file with the following headers: + +- **seedlot** - the name of the Seedlot to associate the event with (must exactly match an existing Seedlot in the database) +- **type** - the name of the Seedlot Maintenance Event type (these vary between Breedbase instances, a list of supported event types is displayed on the upload page) +- **value** - the value of the Seedlot Maintenance Event (these may be different for each event type and vary between Breedbase instances, a list of supported event values is displayed on the upload page) +- **notes** - optional, additional notes/comments about the event +- **operator** - the username of the Breedbase user that recorded the event +- **timestamp** - the date/time the event was recorded, in 'YYYY-MM-DD HH:MM:SS' format + +Once you have an Excel file with the events filled out, follow these steps to upload the events to the database: + +1. Make sure you are logged in to your Breedbase instance +2. Go to the Manage > Seed Lots page +3. Select the **Seedlot Maintenance** button +4. Select the **Upload Maintenance** button +5. Choose your Excel (.xls or .xlsx) file to upload +6. Select the **Upload** button + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_maintenance_upload.png') +``` +*The Seedlot Maintenance upload dialog, showing the supported event types and values (for sugar kelp)* + +#### Recording Events on Website {-} + +To add individual Seedlot Maintenance Events to the database in real time, as they're being recorded, use the **Record Maintenance** page. Follow these steps to record Seedlot Maintenance Events: + +1. Make sure you are logged in to your Breedbase instance +2. Go to the Manage > Seed Lots page +3. Select the **Seedlot Maintenance** button +4. Select the **Record Maintenance** button +5. Enter the **Seedlot Name** or scan a barcode that has the Seedlot Name encoded. Once entered, the box at the top of the page will display basic information about the Seedlot as well its recently recorded events. +6. Select or Enter the values of individual events +7. Optionally, notes button next to each event to add additional notes/comments about that specific event +8. Make sure the operator/username and timestamp are correct +9. Select the **Submit** button to add the recorded events to the database. NOTE: any events that remain selected as "Not Recorded" will not be submitted to the database. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_maintenance_record.png') +``` +*The Seedlot Maintenance record page, as configured for sugar kelp* + +### Displaying Events + +Recently recorded Seedlot Maintenance Events are displayed in a table from the main Seedlot Maintenance page, as well as the detail page for individual Seedlots. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_maintenance_events_unfiltered.png') +``` +*Unfiltered table of recent Seedlot Maintenance events* + +The events displayed in these tables are sorted by timestamp, with the most recently recorded events displayed first. The displayed events can be filtered using any number of supported filter criteria, such as: +- seedlot names (as entered on the page or using an existing seedlot list), +- dates (on, on or before, before, on or after, and/or after the entered dates) +- event types +- event type values +- operator/username + +Select the properties of the filter(s) you want to apply, then select the **Add** button next to the button to add the filter to the list of applied filters. Once you're done adding filters, select the **Filter** button to search the database for the filtered events. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/seedlot_maintenance_events_filtered.png') +``` +*A filtered table of Seedlot Maintenance events* + +The filtered events can be downloaded directly from the table using the **Excel** or **CSV** buttons at the top of the table. Or Seedlot Maintenance Events can be bulk-downloaded (this includes all events for a Seedlot) using a list of Seedlots from the main downloads page (see below). + +### Downloading Events + +To bulk-download all events for a specific subset of Seedlots: + +1. Create a list containing the Seelots you are interested in. +2. Go to the **Download Using Lists** page (Manage > Download) +3. Find the **Download Seedlot Maintenance Events** section +4. Select your list of Seedlots +5. Select the **Download** button to generate the download file + +The downloaded file will follow the same format as the upload template and will contain all recorded Seedlot Maintenance Events for each Seedlot in the list. + +## Deleting Seedlots + +Seedlots can be deleted on the Manage Seedlots page (/breeders/seedlots) by search the seedlot and then clicking the X to delete one seedlot at a time. To delete a seedlot, the logged in user needs the required delete privileges on the seedlot. The seedlot also should not have any transactions associated with it (except for the initial transaction). + +To delete seedlots in bulk, generate a list of type seedlot, for example, using the wizard. Open the section "Delete seedlots using a list" on the Manage Seedlots page and select the list. Seedlot deletion using a list is only available to user with curator status. diff --git a/docs/r_markdown_docs/managing_sequence_metadata.Rmd b/docs/r_markdown_docs/managing_sequence_metadata.Rmd new file mode 100644 index 0000000000..17482e04ce --- /dev/null +++ b/docs/r_markdown_docs/managing_sequence_metadata.Rmd @@ -0,0 +1,67 @@ +# Managing Sequence Metadata + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/sequence_metadata_manage.png') +``` + +## What is Sequence Metadata? + +Sequence Metadata is a feature that allows for the efficient storage and retrieval of sequence annotations for a specific region along a reference genome. The annotation data can contain a primary "score" value and any number of secondary key/value attribute data. For example, Sequence Metatadata can store MNase open chromatin scores for every 10 basepairs along the reference genome as well as genome-wide association study (GWAS) statistics, including the trait information associated with the result. This data can then be filtered by position and/or scores/attribute values and even cross-referenced with markers stored in the database. + + +## Loading Sequence Metadata + +Sequence Metadata can be loaded into the database using a gff3-formatted file. The following columns are used to load the data: + +- **#1 / seqid:** The name of the database feature (ie chromosome) the metadata is associated with (The feature name must already exist as a feature in the database) +- **#4 / start:** The metadata's start position +- **#5 / end:** The metadata's end position +- **#6 / score:** (optional) The primary score attribute of the metadata +- **#9 / attributes:** (optional) Secondary key//value attributes to be saved with the score. These should be formatted using the gff3 standard (key1=value1;key2=value2). The attribute key cannot be either score, start, or end. + +To upload the gff3 file: + +1. Go to the **Manage > Sequence Metadata** page +2. Click the **Upload Sequence Metadata** button +3. On Step 2 of the Wizard, select the Type of data to be uploaded + - This groups similar datasets together in the same Data Type category +4. On Step 3 of the Wizard, select an existing Protocol or create a new one + - The Protocol is used to describe how the data was generated and define the score value and any secondary attributes. Adding the attributes (and their descriptions) to the Protocol will allow the Sequence Metadata queries to filter the data based on the value of one or more of these attributes. Attributes not defined in the Protocol will still be stored and displayed on retrieval, but will not be able to be used in a search filter. +5. Finally, select and upload your gff3 file to the database. The database will verify the format of the file before its contents are stored. + + +## Searching Sequence Metadata + +To retrieve stored Sequence Metadata, go to the **Search > Sequence Metadata** page. + + +### Basic Search + +The basic Sequence Metadata search options include selecting the reference genome and species, the chromosome, and (optionally) the start and/or end position(s) along the reference genome. In addition, one or more specific protocols can be selected to limit the results. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/sequence_metadata_search_basic.png') +``` + +The Sequence Metadata search results are returned as a table, including the chromosome and start/stop positions of the annotation, along with the primary score value and any additional key/value attributes. The markers column will include a list of marker names of any stored markers that are found within the start/stop positions of the Sequence Metadata. The data can be downloaded as a table in an Excel or CSV file or a machine-readable (code-friendly) JSON file. If the Sequence Metadata JBrowse configuration is set, the filtered results can be displayed as a dynamic JBrowse track. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/sequence_metadata_search_results.png') +``` + +### Advanced Search + +Any number of advanced search filters can be applied to the query. The advanced filters can limit the search results by the value of the primary score and/or any of the secondary attribute values. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/sequence_metadata_search_advanced.png') +``` + + +## Marker Integration + +A table of Sequence Metadata annotations are embedded on the Marker/Variant detail page. The table will include any annotations that span the poisiton of the marker (for data of the same reference genome and species). + +## Sequence Metadata API + +A publicly accessible RESTful API (Application Programming Interface) is available to query the database for Sequence Metadata directly from your programming environment (R, python, etc) to be used in analysis. The data is returned in a JSON format. Documentation for the API can be found on the **Manage > Sequence Metadata** page diff --git a/docs/r_markdown_docs/managing_spectral_data.Rmd b/docs/r_markdown_docs/managing_spectral_data.Rmd new file mode 100644 index 0000000000..5922741d80 --- /dev/null +++ b/docs/r_markdown_docs/managing_spectral_data.Rmd @@ -0,0 +1,50 @@ +# Managing Spectral Data + +Breedbase has implemented a flexible spectral data storage protocol that handles spectral data irrespective of the source spectrometer. Spectral data storage and analysis in Breedbase makes use of the R package *waves* for outlier identification, plotting, sample aggregation, and prediction model training. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/waves_breedbase_schema.png') +``` + +--- +## Upload Spectral Data +Spectral data can be added as a CSV file that includes metadata in the leftmost columns followed by one column per spectral measurement to the right. Rows represent a single scan or sample, each with a unique ID that must match to a Breedbase observationUnitName. Future data transfer using [BrAPI](https://brapi.org) will allow for interoperability with data collection software. + +To upload a spectral dataset, navigate to the 'Manage NIRS Data' page by selecting 'NIRS' in the 'Manage' menu and click the blue 'Upload NIRS' button. This will open an upload workflow. A link to the required file format and an example .csv file can be found by clicking in the light blue info box in this workflow. Another example of the file format is shown below. + +* **id**: Optional identifier for each NIRS read. The id must be an integer. +* **sampling_id**: Optional identifier for each sample. Strings are allowed. +* **sampling_date**: Optional field. The format allowed is: YYYY-MM-DD. +* **observationunit_name**: Required field that matches existing data in the database. It can be the plot name, subplots, plant name, or tissue sample, depending how your trial is designed. +* **device_id**: Optional field to identify your device. Strings are allowed. +* **device_type**: Required field. It is possible upload data for a single device type. They can be: SCiO, QST, Foss6500, BunchiN500, or LinkSquare. +* **comments**: Optional field for general comments. +All other columns are required wavelengths. You can add how many columns you want upload -- there is no limit. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_NIRS_main_display.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/upload_NIRS_example_format.png') +``` + +## Evaluate and Remove Outliers +Spectral calibration models can be heavily affected by the presence of outliers, whether they come from spectrometer spectral artifacts or user errors. Mahalanobis distance (Mahalanobis, 1936) is a measure of the distance between a single observation and a larger distribution and is commonly used in the identification of outliers in a multivariate space (Des Maesschalck et al, 2000). The *FilterSpectra()* function in the R package [*waves*](https://CRAN.R-project.org/package=waves) calculates the Mahalanobis distance of each observation in a given spectral matrix using the *stats::mahalanobis()* function. Observations are identified as outliers if the squared distance is greater than the 95th percentile of a $\chi$^2^-distribution with *p* degrees of freedom, where *p* is the number of columns (wavelengths) in the spectral matrix (Johnson and Wichern, 2007). In Breedbase, this procedure is applied on a per-dataset basis on upload and outliers are given binary tags "Outlier." + +## Plot Spectra +After outlier identification, a plot is generated using the *PlotSpectra()* function in [*waves*](https://CRAN.R-project.org/package=waves). This function uses the filtered spectra and *ggplot2::ggplot()* to create a line plot with outliers highlighted by color. A list of rows identified as outliers are shown beneath the plot. Plots are saved as .png files and linked to the original input datasets. Plot image files can be downloaded via the "Download Plot" button in the upload workflow. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/upload_NIRS_plot_example.png') +``` + +## Aggregate Spectra +To obtain a stable and reliable spectral profile, most spectrometer manufacturers recommend that multiple spectral scans are captured for each sample. While some spectrometers aggregate these scans internally, many do not, requiring the user to do so before analysis can take place. Breedbase handles these cases upon data upload following filtering steps by calling the *AggregateSpectra()* function from [*waves*](https://CRAN.R-project.org/package=waves), saving the aggregated scans for future access through the search wizard feature. Scans are aggregated by sample mean (e.g. plot-level basis) according to the provided observationUnitName field. After aggregation, the user exits the upload workflow and the raw data file is saved in the upload archive. + +## References +* De Maesschalck, R., Jouan-Rimbaud, D., and Massart, D. L. (2000). The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. +* Johnson, R. A. \& Wichern, D. W. (2007). Applied Multivariate Statistical Analysis (6th Edition). p 773. +* Mahalanobis, P. C. (1936). On the generalized distance in statistics. National Institute of Science of India. + +[**Analysis tool documentation**](#spectral-analysis) diff --git a/docs/r_markdown_docs/managing_tissue_samples.Rmd b/docs/r_markdown_docs/managing_tissue_samples.Rmd new file mode 100644 index 0000000000..b4e60c111c --- /dev/null +++ b/docs/r_markdown_docs/managing_tissue_samples.Rmd @@ -0,0 +1,69 @@ +# Managing Tissue Samples + +To access this page go to Manage and then Tissue Samples. + +## Tissue samples from field trials + +A field trial contains plots planted with a specific accession. Each plot can contain many plants, which in turn can contain many tissue samples. +On the manage tissue sample page we can see the field trials that contain tissue samples already. We can choose to download the tissue sample layout as seen in the below picture. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_download.png') +``` + +If the field trial you want to collect tissue samples from is not in the above table, you can click the button highlighted below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_button.png') +``` + +Once you have clicked this button, you will enter a workflow that begins with the following introduction. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_intro.png') +``` + +Once you click next, you will need to select your trial. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_select.png') +``` + +Next, if your trial currently only has plot entries saved, you will be asked to enter how many plants are in each plot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_num_plants.png') +``` + +Finally you will be asked how many tissue samples you want for each plant. You can specify a string to include in the tissue sample name, such as leaf or root. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_num_tissues.png') +``` + +Afterwards you should see the following success message, indicating that the tissue samples are saved. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_field_trial_submit.png') +``` + +## Genotyping Plate Tissue Samples (96 or 384 well plates) + +A genotyping plate represents a 96 or 384 well plate. You can use the Coordinate Android application to create your plate layout, or you can upload your own Excel plate layout, or you can use the database to generate a plate layout. +Ideally, you will use tissue sample names originating from a field trial as the "source" for each well tissue sample, but you can also use plant names, plot names, or accession names. + +From the manage tissue samples page, you can see the genotyping plates saved in the database. You can also download the layouts as shown below. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_genotyping_trial_download.png') +``` + +If you need to create a new genotyping plate, you can click the button shown below. This will guide you through a workflow for uploading or creating the new plate layout. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_tissue_samples_create_tissues_genotyping_trial_button.png') +``` + +Genotyping vendors require you to send a plate layout during submission. You can download the plate layout as shown above, or you can go to a genotyping plate detail page to download the Intertek formatted file. + +In the future you will be able to directly export your genotyping plate plate layout to vendors. diff --git a/docs/r_markdown_docs/managing_user_roles.Rmd b/docs/r_markdown_docs/managing_user_roles.Rmd new file mode 100644 index 0000000000..0a6a14195a --- /dev/null +++ b/docs/r_markdown_docs/managing_user_roles.Rmd @@ -0,0 +1,22 @@ +# Managing User Roles + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_user_roles_page.png') +``` + +## What are User Roles? + +Every user account in Breedbase has one or more associated "roles" that determine the authorizations (what the user is allowed to do) in the database. There are three fundamental roles, "curator", "submitter", and "user", which determine basic read/write levels. The "curator" status can read and write everything in the database. The "submitter" status can add information and edit or delete previously submitted information. The "user" type can only read data. Additional roles represent the breeding programs, and are sometimes used to fine-tune write and edit capabilities, as it necessary for multiple users in a breeding program to edit each other's data. + +## The Manage User Roles page + +In the "Manage" menu, select the item "User Roles". This will show the current users in the database with their associated roles. If you are logged in as a curator, the table will show system roles as well as breeding program roles; if you are logged in as a submitter or user, it will show breeding program membership. + +If logged in as a "curator", the roles can be added or deleted. + +- To delete a role, click on the X in the role name. A confirm dialog will be displayed to prevent accidental deletion. +- To add a role, click on the plus sign next to the roles. A dialog will pop up with a list of roles. Select the desired role and click "Submit". +- The new role should be displayed next to the user immediately. +- Role deletions and additions will be effective immediately. + +It is recommended that few users be given the "curator" privileges to avoid confusion over data ownership and accidental data overwriting and deletion. diff --git a/docs/r_markdown_docs/managing_vcf_data.Rmd b/docs/r_markdown_docs/managing_vcf_data.Rmd new file mode 100644 index 0000000000..522016283f --- /dev/null +++ b/docs/r_markdown_docs/managing_vcf_data.Rmd @@ -0,0 +1,95 @@ +# Managing VCF Data + +## Uploading VCF Data + +Genotyping data in VCF can be loaded from the web-interface. Breedbase can store any genotypic variants from a VCF, allowing for polyploids, structural variants, etc. without problems. + +To begin go to Manage->Genotyping Plates and click the button seen below: +Note that you do not need to have genotyping plates uploaded to upload VCF data; you may upload genotyping data to accessions or you can upload genotyping data for tissue samples in genotyping plates. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_upload_button.png') +``` + +The workflow begins with an intro: + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_upload_dialog_intro.png') +``` + +On the following step in the workflow, a genotyping project is defined or selected. A genotyping project is a high-level entity for grouping several genotyping events. It is defined with a name, description, name, breeding program, and genotyping facility (IGD, Intertek, etc.). + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_upload_dialog_project.png') +``` + +The following step is to define or select a genotyping protocol. A genotyping protocol represents the set of markers being called against a specific reference genome. A genotyping protocol is defined with a name, description, reference genome name, species name, and a location of data generation. Note in the picture that you can select whether the samples in your file are accessions or tissue samples in the database; tissue samples are for when a genotyping plate is stored in the database. There is an option to parse the sample names for appended sequencing numbers from IGD, where the sample names are like "accession:igdnumber". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_upload_dialog_protocol.png') +``` + +The final step is to select the VCF from your computer and upload it. The web interface can be used to upload files arbitrarily large; it is a NGINX configuration to set this size. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_upload_dialog_vcf.png') +``` + +## Searching and Downloading VCF Data + +The Search Wizard is the primary means of querying data in the database. Go to Search->Wizard to begin. + +Once genotyping protocols are stored, select Genotyping Protocols from the first dropdown menu. Then if you select one or more and select Accessions from the second dropdown menu, you will see the accessions for which genotypes were stored. As seen in the following picture, there is a section for filtering genotypes by chromosome, start position, and end position. Genotypes can be downloaded in VCF or DosageMatrix formats. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_wizard_download_prot.png') +``` + +Using the "Default genotyping protocol" which is configured in a system, you can query over field phenotypic evaluations before downloading genotypes and phenotypes. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_genotyping_data_wizard_trial.png') +``` + +## Searching Protocols + +Genotyping protocols can be search by going to Search->Genotyping Protocols. To download genotypes accessions must be selected, though any combination of search criteria can be used to filter and select those accessions. If a genotyping protocol is not selected, then the default genotyping protocol set in the configuration will be used. Genotyping protocols can also be selected in the wizard. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_wizard_genotyping_protocols_download_main.png') +``` + +The genotyping download menu on the Search Wizard presents options for filtering by chromosome, start position, and end position. Genotypes can be downloaded in VCF of Dosage Matrix formats. The genomic relationship matrix (GRM) can be downloaded for the selected accessions in a tab-delimited matrix format or in a three-column format that is useful in Asreml. Genotypes can be computed from the parents in the pedigree if those parents are genotyped by clicking on the "compute from parents" checkbox. Additionally, the GRM can be computed using genotypes of parents in the pedigree if the "compute from parents" checkbox is selected. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/search_wizard_genotyping_protocols_download_genotypes.png') +``` + +As is described elsewhere, the Search Wizard presents a way to filter phenotypic values by minimum and maximum values, and allow for download in CSV and Excel formats. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/search_wizard_genotyping_protocols_download_phenotypes.png') +``` + + +## Detail Pages and Deletion + +The genotyping protocol detail page will show all information about the protocol such as the reference genome used, the header information lines in the uploaded VCF file, the markers involved, and the samples genotyped. + +The markers section will show all markers used and their annotations, such as position, chromosome, alternate allele, reference allele, marker format, etc. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/genotyping_protocol_detail_page_markers.png') +``` + +The samples section will show all samples genotyped. Notice the Download links in the table which can be used to easily get the VCF file results for each genotyped samples with all markers in the genotyping protocol. For getting mulitple samples at once, use the Search Wizard as discussed above. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/genotyping_protocol_detail_page_samples.png') +``` + +The genotyping protocol and all associated genotyping data can be deleted from the genotyping protocol page. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/genotyping_protocol_detail_page_delete.png') +``` diff --git a/docs/r_markdown_docs/searching_the_database.Rmd b/docs/r_markdown_docs/searching_the_database.Rmd new file mode 100644 index 0000000000..6b6597fb68 --- /dev/null +++ b/docs/r_markdown_docs/searching_the_database.Rmd @@ -0,0 +1,200 @@ +# Searching the Database + +You can search for information on the database by using the following search options: Wizard, which uses combined criteria specified by users; Accessions and Plots; Trials; Markers; Images; People; FAQ. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/image267.png') +``` + +## The Search Wizard {#search-wizard} + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_interface.png') +``` + +### How the Search Wizard Works + +The search wizard presents a number of select boxes, which are initially empty. You start searching by picking a category of data from the dropdown above the left-most select box. + +Once a category has been picked, the database will retrieve all the options within this category and display them within the first select box. You then select one or more options from the first select box, which activates the second dropdown. + +You can then select a category from the second dropdown, and repeat this same search process through all four dropdowns and select boxes. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_interface_selections.png') +``` + +- In the example above, the "locations" category was chosen in the first dropdown. The first select box then displayed all the possible locations in the database. The option Ibadan was selected. + +- This activated the second dropdown. The category “years” was chosen in the second dropdown. The second select box then displayed all the years that are linked in the database to the location Ibadan. From that list, the options 2011 and 2012 were selected. + +- This activated the third dropdown. A final category, “accessions”, was chosen in the third dropdown. The third select box was then populated with the 3847 accessions in the database that are linked with the location Ibadan in the years 2011 or 2012. + +In addition to the basic search operations demonstrated above, users can take advantage of two more features: + +**Load Selection from List** + +```{r echo=FALSE, out.width='25%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_select_list.png') +``` + +- Instead of picking a category in the first dropdown, users can instead populate the first selectbox from a list by scrolling down in the first dropdown to the "Load Selection from List" subheading and selecting a list. This is useful for starting queries with a list of plots, as this category is not among the options in the first dropdown. + +**ANY/MIN/ALL** Toggle + +```{r echo=FALSE, out.width='25%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_any_min_all_toggle.jpg') +``` + +- By default, the search wizard combines options within a category using an OR query. In the example above, in the third panel the wizard retrieved accessions associated with the location 'Ibadan' in **ANY** of the years "2011 **OR** 2012" + +- If the user clicked the toggle below the second select box to change it to **ALL** before choosing accessions in the third dropdown, the wizard would instead retrieve accessions associated with the location 'Ibadan' in the years "2011 **AND** 2012". This will be a smaller set of accessions, because any accessions used only in 2011, or only in 2012 will be excluded. + +- A more advanced search could use the **MIN** toggle option. This allows the user to make a query in between an ANY or ALL query, where a minimum number of matches from the selected column will be used as a filter for the next column. The minimum can be provided as either a percentage (%) or an actual count of items (#). In the example above, if the years 2011, 2012, and 2013 were selected in the second column, the user could enter '2' in as the minimum and select '#' as the minimum match type. This would select accessions in the third column that were used in 2 or more of the selected years.
    + +```{r echo=FALSE, out.width='25%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_any_min_all_toggle_min_details.jpg') +``` + +### How to use retrieved data + +#### Getting more Info {-} + +Any option in the wizard select boxes (except for years) can be clicked to open a page with more details. The new page is opened in a new tab. + +#### Saving to a list {-} + +You can store the highlighted items in any selected box to lists. This is done using the inputs and buttons directly below the select box. **Don’t forget, you must be logged in to work with lists!** + +```{r echo=FALSE, out.width='50%', fig.align='center'} +knitr::include_graphics('assets/images/add_create_list.png') +``` + +- To **add items to an existing list**, first pick an existing list using the "Add to List..." dropdown on the left. Then click the "Add" button. A popup window will confirm the action, and display the number of items added to your existing list. + +- To **store items to a new list**, first type a new list name in the "Create New List..." text input on the left. Then click on the "Create" button. A popup window will confirm the action, and display the number of items added to your new list. + +#### Downloading Data {-} + +You can download trial metadata, phenotypes and genotypes associated with the highlighted items in the wizard select boxes. This is done using the buttons in the download section at the bottom of the page. **Don’t forget, you must be logged in to download data!** + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_download_options.png') +``` + +##### Metadata {-} + +Trial metadata can be downloaded by selecting a subset of trials from the database or based on your search categories. To download, click on "Related Trial Metadata", a dialog will appear. Select download format and click the "Metadata" button to complete your download. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_related_metadata_download.png') +``` + +##### Phenotypes {-} + +The phenotypes download is quite flexible, and can download a subset of all the trial data in the database based on whichever categories and options you currently have selected. Simply click on the “Related Trial Phenotypes” link, review the options, changing or adding any additional parameters you like, then click ‘Download Phenotypes’. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_related_phenotypes_download.png') +``` + +##### Genotypes {-} + +The genotype download is more stringent. It requires a minimum of one accession and one genotyping protocol to be selected in the wizard select boxes. The text box in the download section of the page will help track what has been selected. Once clicked, the “Download Genotypes” button will download a genotype file for the selected accessions. + +#### Saving the wizard selections {-} + +As discussed above, the selections of the individual select boxes in the wizard can be saved separately to a list. The lists can be used as inputs in other tools on the site. However, sometimes creating a selection is quite time consuming and restoring the selections from four different lists would be cumbersome too. Therefore, the selections can be saved together in a dataset, and named for later retrieval. This is done in the section "Load/Create Datasets" that is below the first two wizard select boxes. To select an existing dataset, one uses the "Load Dataset" dropdown. A particular dataset can be chosen, and the "Load" button can be clicked to retrieve and display the dataset in the wizard. To create a new dataset using items that are selected in the wizard, one can enter the name of the new dataset in the "Create New Dataset" text box. Once the dataset has been given a name, clicking the "Create" button will save the new dataset. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_load_create_dataset.png') +``` + +### Updating the Wizard + +The search wizard uses a copy of the database, or a cache, to return results quickly. If data appears to be missing, it usually means that the cache needs to be updated. Users with submitter privileges or above can do this using the ‘Update Wizard’ button. One can also use the ‘Refresh Lists’ button to update the available lists. + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/wizard_update_list_refresh.png') +``` + +This will take just a few seconds in small databases, but may take a few hours to complete in larger databases. + +## Accessions and Plot Search + +Accessions and their related materials (cross, plant, plot, population, tissue\_sample, training population) can be searched by using “Search Accessions and Plots” page. On this page, “accession” is the default stock type; however, you can change stock type by selecting an option from the drop-down list. +From this page you can construct detailed queries for stock types. For example, by using the "Usage" section, the "Properties" section, and the "Phenotypes" section you could search for accessions which were diploids used in a specific year and location and were also phenotyped for height. You can also search for accessions based on genetic properties, such as the location of an introgression on a specific chromosome. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_accessions.png') +``` + +It is possible to query over any of the available properties, such as "ploidy_level", "country of origin", "introgression_chromosome", etc. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_accessions_properties_search.png') +``` + +In the search result table it is possible to select any of the available properties to view. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_accessions_properties_view.png') +``` + +At the bottom of the accession search there is a phenotype graphical filtering tool. Here you can filter down accessions based on combinations of trait performance. The filtered down accessions are then able to be saved to a list. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_accessions_graphical_filtering.png') +``` + +For information on adding Accessions please see the Managing Accessions help. +For information on how field trial plots, plants, tissue samples, and subplots are added to the database, please see the Managing Field Trials help. + +## Trials Search + +Trials on the database can be searched based on trial name, description, breeding program, year, location, trial type, design, planting date, and harvest date. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trial_search.png') +``` + +## Trait Search + +On the Trait Search page (menu item `Search > Traits`), traits in the database can be searched by ID, name, or descripiton. Optionally, a starting list of traits can be selected to filter down results. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trait-search-default.png') +``` + +Selecting traits in the results of the search allows one to add the selected results to a trait list, or create a new trait list from the select results. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/trait-search.png') +``` + +## Ontology Browser + +A more advanced tool for searching for Traits is the ontology browser, available by clicking on Analyze and Ontology Browser. From here you can search ontologies and see the various classifications of terms in a tree display. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/ontology_browser.png') +``` + +The terms which appear in the Trait Search in 2.4 are only variable terms. The ontology browser shows these variables as different from their grouping terms by indicating VARIABLE_OF like in the following screenshot. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/ontology_browser_variable.png') +``` + +## Search Seedlots + +Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. To search for available seedlots you go to Manage and then click Seed Lots. By clicking Search Seedlots, you can specify query information. The results from your search will be in the table below the search form. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/search_seedlots.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/manage_seedlots.png') +``` + diff --git a/docs/r_markdown_docs/static/documents/img/Breedbase_HighRes.png b/docs/r_markdown_docs/static/documents/img/Breedbase_HighRes.png new file mode 100644 index 0000000000..327f8a40aa Binary files /dev/null and b/docs/r_markdown_docs/static/documents/img/Breedbase_HighRes.png differ diff --git a/docs/r_markdown_docs/using_fieldbook_app.Rmd b/docs/r_markdown_docs/using_fieldbook_app.Rmd new file mode 100644 index 0000000000..ff18310c45 --- /dev/null +++ b/docs/r_markdown_docs/using_fieldbook_app.Rmd @@ -0,0 +1,213 @@ +--- +output: + pdf_document: default + html_document: default +--- +# Using Field Book and BreedBase for Data Collection and Storage {#Using-Field-Book-with-BreedBase} + +```{r echo=FALSE, out.width='75%', fig.align='center'} +knitr::include_graphics('assets/images/FB_Section_Header.png') +``` + +## Introduction + +**Field Book** is an open-source Android application designed to streamline phenotypic data collection, replacing traditional paper field books and reducing transcription errors. + +**BreedBase** supports integration with Field Book by enabling users to generate properly formatted field layout and trait files, import collected data, and optionally exchange data via **BrAPI**. + +This section provides step-by-step guidance on preparing Field Book for data collection using trial data stored in BreedBase, including file generation, manual file import, and data export. It assumes that trial fields and trait definitions already exist in BreedBase and that users have access to an Android device. + +[*Field Book User Manual*](https://fieldbook.phenoapps.org/#/) + +--- + +## Installing Field Book + +Field Book can be installed on an Android device via the Google Play Store or manually from GitHub. To ensure access to the latest features and fixes, we recommend checking for updates regularly. + + +### Option 1: Downloading from the Google Play Store + +Download the Field Book app directly from the [Google Play Store](https://play.google.com/store/apps/details?id=com.fieldbook.tracker) or scan the QR code below. + +```{r echo=FALSE, out.width='25%', fig.align='center'} +knitr::include_graphics('assets/images/QR_FieldBook_PlayStore.png') +``` + +[**Walkthrough: Installation and Setup**](https://scribehow.com/shared/Fieldbook_Installation_and_Initial_Setup__3E-cowQMRcOKhfRl_sqFDQ) + + + +### Option 2: Manual Installation from Github + +[**Walkthrough: Installing Field Book from GitHub**](https://scribehow.com/shared/Installing_Field_Book_from_GitHub__M0qchs3uThGb_ms2nDRg3A) + + + +### Keeping Field Book Updated + +Regular updates are recommended to access new features and bug fixes. + +[**Walkthrough: Updating the Field Book App**](https://scribehow.com/shared/Updating_the_Field_Book_App__qdTEMk1lSkadYj8zXgOYgA). + + + +--- + +## Preparing Field Layout Files in BreedBase + +Field Book requires a **field layout file** containing a unique identifier for each plot or plant to be observed. Optionally, columns for row, column, pedigree, or previous phenotype data may be included for context. + +BreedBase provides tools to generate layout files directly from trial metadata. These files can be exported and then uploaded to Field Book for data collection. + +[**Walkthrough: Creating a Field Layout File in BreedBase**](https://scribehow.com/shared/Creating_a_Field_Layout_File_for_Field_Book_in_Breedbase__FDBwW83OS66kHXIvVsP-1A?referrer=documents) + +--- + +## Importing Field Layout Files into Field Book + +Field layout files can be imported into Field Book via cloud storage or directly from the device. + +### Option 1: Import from Cloud Storage + +Requirements: + +1. A generated field layout file. +2. Linked cloud service (e.g., Google Drive) on the device. +3. File uploaded to the cloud account. + +[**Walkthrough: Importing from Cloud Storage**](https://scribehow.com/shared/Importing_Field_Layout_Files_to_Field_Book_from_Cloud_Storage__5JLV59I8TzCgDeTkUPbIdA) + + + +### Option 2: Import from Local Storage Device + +Requirements: + +1. A generated field layout file. +2. File transferred to the Android device’s local storage. + +[**Walkthrough: Importing from Local Device**](link to be added) + +--- + +## Adding Traits to Field Book Using BreedBase + +For exported data to be compatible with BreedBase, trait names in Field Book must match the exact names stored in the database. + +BreedBase supports generation of **trait files** containing properly formatted trait names, types, units, and metadata. + +### Creating Trait Files in BreedBase + +[**Walkthrough: Creating Trait Files in BreedBase**](https://scribehow.com/shared/Creating_a_Field_Book_Trait_File_in_BreedBase__AQKYEuyXRzK1H1lRFm--3Q) + + + +### Importing Trait Files to Field Book + +[**Walkthrough: Importing Trait Files**](https://scribehow.com/shared/Importing_Trait_Files_into_Field_Book__LuVCpgtfQCeDtWEl7xbZVw) + + + +--- + +## Collecting and Exporting Phenotype Data + +This section describes the process for recording and exporting phenotype data using the Field Book application. It assumes that the required field layout and trait files have already been imported into the app. + +Users will be guided through entering phenotypic observations in the field and exporting collected data in a format compatible with downstream processing and upload to **BreedBase**. This standardized workflow helps ensure data integrity, traceability, and seamless integration with breeding databases. + +### Collecting Phenotype Data in Field Book + +Once field layout and trait files are loaded, data collection can begin. Field Book allows users to view each plot or plant entry, record trait values in structured forms, and automatically timestamp and geotag observations (if device settings allow). + +[**Walkthrough: Collecting Data in Field Book**](https://scribehow.com/shared/Collecting_Data_in_Fieldbook__hIsZkb9xSnKcLW65gN60lA) + + +### Exporting Phenotype Data from Field Book and Importing to BreedBase + +After data collection is complete, Field Book allows users to export data as a .csv file. This file can then be uploaded to BreedBase for storage and analysis. + +[**Walkthrough: Exporting Phenotype Data from Field Book**](https://scribehow.com/shared/Exporting_Phenotype_Data_in_FieldBook__ToUuuA3YTYGclEg7SDArLg) + +Once exported, the .csv file can be imported into BreedBase through the trial’s data upload interface. Proceed with uploading phenotype observations as described in the **Managing Phenotypic Data** section. + +It is important that: + +- Trait names match those defined in the database. +- Plot or plant identifiers align with the original layout. + +--- + +## Using BrAPI for Data Transfer Between Field Book and BreedBase + +Field Book supports integration with **BreedBase** via the **BrAPI (Breeding API)** specification. BrAPI enables direct exchange of trial metadata and phenotype observations between Field Book and BreedBase without the need for manual file uploads. + +This walkthrough covers: + +- Connecting Field Book to a BrAPI-enabled BreedBase server +- Importing trial layout and traits from BreedBase to Field Book +- Exporting collected phenotype data from Field Book to BreedBase + +**Prerequisites:** + +- You must have a BreedBase user account with permissions to access trial and trait data. +- Your BreedBase instance must be configured with working BrAPI endpoints. +- The Field Book app must be installed on an Android device with internet access. + +**Advantages of Using BrAPI:** + +- Reduces manual errors from file handling. +- Supports real-time syncing of data. +- Enables remote team coordination and efficient workflows. + +### Connecting Field Book to a BrAPI Server + +[**Walkthrough: Connecting Field Book to a BreedBase BrAPI Server**](https://scribehow.com/shared/Enable_BrAPI_for_Field_Book_using_the_Scannable_BrAPI_Configuration__SUXsC8xKQX-rLEG-4H6BIA) + + + +### Importing Trials and Traits from BreedBase + +[**Walkthrough: Importing Trials from BreedBase via BrAPI**](https://scribehow.com/shared/Import_Field_Trials_from_BreedBase_via_BrAPI__R4munb3fQ-OO5f-WAZRnKg) + + + +[**Walkthrough: Importing Traits from BreedBase via BrAPI**](https://scribehow.com/shared/Import_Traits_from_BreedBase_via_BrAPI__aqvmCO8QS_eziilPrntOoA?referrer=documents) + + + +### Collecting Data + +Proceed with entering phenotype observations as described in the **Collecting Phenotype Data in Field Book** section. + +Ensure that: + +- Trait values match expected formats (e.g., numeric, categorical). +- Observations are saved regularly during field use. + +### Exporting Phenotype Data to BreedBase via BrAPI + +[**Walkthrough: Sending Phenotype Data with BrAPI**](https://scribehow.com/shared/Exporting_Data_with_BrAPI__PaWGH98ESQWGrXIZk5lJTQ) + + + +### Verifying Data in BreedBase + +After exporting data to BreedBase via BrAPI, it is important to verify that the export was successful directly in BreedBase. + +1. Log in to your BreedBase instance. +2. Navigate to the Manage Trials or Trial Detail page for the uploaded study. +3. Confirm that: + - Trait data appears in the Phenotype Data tab. + - Data points align with expected plot/plant identifiers. + - Trait units and scales match those defined in the database. + + +### Troubleshooting + +- Connection fails: Ensure you have the correct BrAPI server URL and valid credentials. +- Traits do not load: Confirm that the trait exists in BreedBase. +- Upload fails: Check for internet connectivity, correct user permissions, and trait formatting errors. + + diff --git a/docs/r_markdown_docs/using_label_designer.Rmd b/docs/r_markdown_docs/using_label_designer.Rmd new file mode 100644 index 0000000000..33ff48806a --- /dev/null +++ b/docs/r_markdown_docs/using_label_designer.Rmd @@ -0,0 +1,66 @@ +# Using the Label Designer + +Breedbase provides an interactive design tool for creating custom labels. To access the label design tool, click on “Label Designer” in the “Manage” menu. The following sections explain your many options as you advance through each step of the design workflow. + +## Select a Data Source + +The first step is to select a data source. To do so, you may directly select a specific data source from the "Data Source" dropdown menu (either a field trial, genotyping trial, or crossing trial) to populate your labels with the trial design information. Alternately, from the same dropdown menu, you may select a list to populate your labels with the list contents. Prior to making a selection from the "Data Source" dropdown menu, you may also filter the data sources by using the "Datatype" dropdown menu. +You must then choose a level (plot, plant, etc.) before proceeding, using the "Label for Every" dropdown menu. To generate plot-level labels for more than one trial at once, select a list of trials as the source and plot as the level. Once you have made your selections, click the 'Next' button to move to the next step in the workflow. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +if (knitr:::is_latex_output()) { + knitr::include_graphics('assets/gifs/gif1.png') +} else { + knitr::include_graphics('assets/gifs/gif1.gif') +} +``` + + +## Set Page and Label Size + +After selecting the data source, you must choose whether to create a new design or load a saved design. If you choose the "New" selector, you will be prompted to select a page size and label size. +If you do not see your page size or label size as an option, then select "Custom" from the relevant dropdown menu and enter your desired dimensions in pixels (each pixel is 1/72nd of an inch). +If you choose the "Saved" selector, you will be prompted to select a saved design. After selecting a saved design, you will be taken directly to the design step with the saved design elements pre-loaded. + + +```{r echo=FALSE, out.width='95%', fig.align='center'} +if (knitr:::is_latex_output()) { + knitr::include_graphics('assets/gifs/gif2.png') +} else { + knitr::include_graphics('assets/gifs/gif2.gif') +} +``` + + +## Design Your Label + +Having set the page and label formats, you will be presented with a label canvas where you can begin adding elements to your label. Select a type, field, size, and font, then click "Add". +You can add text to an existing field or create a completely custom field by clicking "Create Custom Field". +Once added, you can drag and drop elements, or you can delete them by clicking on the red box in their upper left corners. +Barcodes can also be resized by dragging the green box in their lower right corners. +If you are creating labels for a trial, it is highly recommended that you include a barcode encoding your plot, plant, or tissue sample names. +These are the unique identifiers that will need to be included with any phenotypic or genotypic measurements loaded into the database. +When you are satisfied with your label design, click "Next". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +if (knitr:::is_latex_output()) { + knitr::include_graphics('assets/gifs/gif3.png') +} else { + knitr::include_graphics('assets/gifs/gif3.gif') +} +``` + +## Adjust Formatting, Save, and Download + +In the last step of the workflow, you can tweak your formatting and page layout, save your design, or download your labels. +The "Additional Settings" dialog will allow you to adjust the page margins and margins between labels. The units are pixels (each pixel is 1/72nd of an inch). It is not recommended that you change these settings until you have already printed a test page. +You can also set the number of copies per label, filter by replicate, or download only the first page for test purposes. +To save your design, type a unique name in the text field and click "Save". This will save your design to your list manager, where you can make it public in order to share it with others. +Finally, if you are ready, click download to generate and download your labels! +```{r echo=FALSE, out.width='95%', fig.align='center'} +if (knitr:::is_latex_output()) { + knitr::include_graphics('assets/gifs/gif4.png') +} else { + knitr::include_graphics('assets/gifs/gif4.gif') +} +``` diff --git a/docs/r_markdown_docs/vector_viewer.Rmd b/docs/r_markdown_docs/vector_viewer.Rmd new file mode 100644 index 0000000000..7d54c43fd5 --- /dev/null +++ b/docs/r_markdown_docs/vector_viewer.Rmd @@ -0,0 +1,53 @@ +# Vector Viewer + +BreedBase provides a vector viewer tool that can generate, save, or upload vectors onto BreedBase. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/vectorViewer.png') +``` + +The vector viewer tool comes with many interactive features: + +- Highlighting + +- Location display + +- Zooming in / panning out + +- A feature and restriction enzyme table that support real time editing, deleting, or adding + + + + + +To highlight a region of the vector viewer, click and drag your cursor clockwise. To delete the new highlight, either start a new highlighted segment by clicking on the vector, or click the "Update/Reset" button. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/vectorHighlight.png') +``` + +To find the location of a spot on the vector, hover your cursor over the vector and a number will appear. That is the location, in base pairs, of your cursor. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/cursorLocation.png') +``` + +To zoom in and pan out, click the "+" or the "-" button underneath the vector. + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/ZoomIn.png') +``` + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/ZoomOut.png') +``` + +To add a feature or restriction site, click "Add". To edit a feature or restriction site, select the feature or restriction site of choice, click "Edit", and make any wanted changes. To delete a feature or restriction site, selected the feature or restriction site that you want to delete and then click "Delete". + +```{r echo=FALSE, out.width='95%', fig.align='center'} +knitr::include_graphics('assets/images/vectorButtons.png') +``` + + + + diff --git a/docs/search_index.json b/docs/search_index.json new file mode 100644 index 0000000000..3c568bd11d --- /dev/null +++ b/docs/search_index.json @@ -0,0 +1 @@ +[["index.html", "Breedbase User Manual Introduction", " Breedbase User Manual Breedbase team 2026-01-08 Introduction Welcome to the Breedbase manual! Use the table of contents in the left sidebar to navigate to the topic of your choice. At any time you can select specific text in the manual to highlight or annotate it using Hypothesis. Open the Hypothesis sidebar on the right to view existing annotations. You may also use the widgets at the top of the screen to: - collapse the sidebar - search for a specfic topic - change the font size, font type, or the site theme - download the manual as a pdf Manual as a pdf can be download here also. Download This manual is intended for database users. If you are a developer looking for software implementation details, please visit the developer wiki instead: https://github.com/solgenomics/sgn/wiki "],["basic-website-usage.html", "Chapter 1 Basic Website Usage 1.1 Creating a User Account 1.2 Managing your Account 1.3 Menu Layout 1.4 Working with Lists 1.5 User Permissions", " Chapter 1 Basic Website Usage Breedbase is usually hosted on the cloud and is entirely web-based, with only a browser required to access and use it. The recommended browser is Firefox. To use Breedbase, you or your project will usually need to have your own Breedbase instance. Information how you can obtain your own instance can be obtained from the Breedbase project (https://breedbase.org/). Once an instance is set up, the site needs to be configured and some metadata uploaded before you can design, run and analyze trials: define user accounts and their access privileges. add a trait ontology with the traits that you require. add the locations that you use in your breeding program add the foundational germplasm and the respective pedigree data historical trial data can be uploaded as needed. Usually more recent trials are prioritized over older trials In this chapter, we will cover how you can manage the user accounts and some basic website features such as lists and how to navigate the menus. The other topics are covered in subsequent chapters. Note that Breedbase instances are highly customizable, and that not every instance of Breedbase will have the same options in the same location. Refer to the site specific documentation, if any, for site specific information. 1.1 Creating a User Account 1.1.1 Verifying first that you do not already have an account Before creating an account, please verify first that you dont already have an account. You can use Search menu to check if you already registered as a user. In the Search menu, selecting the People tab and search your name. If nothing is found, proceed with the instructions below. Otherwise, continue by clicking the Login button. If you have forgotten your password, you can retrieve it by clicking the Forgot your password? link on the login page. 1.1.2 Creating a user account On the right of the toolbar, click on Login, which will take you to the login dialog. On the login dialog, click on the link sign up for an account. It will take you to the page below: Filling in all of the information, then click Create Account. After you submit the information, an email will be sent to the provided email address. Check your email and click on the link to activate your account. 1.2 Managing your Account 1.2.1 Login To login, click the Login link in the toolbar on any page and enter your username and password. If you have forgotten your password, you can retrieve it by clicking the Forgot your password? link on the login page. 1.2.2 Editing Account Settings Account settings can be edited by clicking on the my profile link displayed as your user name, on the right of the toolbar. You must be logged in order to access and change account settings. You can add personal information to your account using the View or update personal information link. To change your password, username, or your contact email, click on Update account information link. You must provide your old password before you can make any changes. 1.2.3 Changing Your Account Status: From User to Submitter After you create an account, your account has a user status. This account has limited privileges. Accounts with user status are able to: Change personal information Post comments on pages Post to the forum To upgrade your account status to submitter, contact the database curators using the contact link provided at the footer of each page. Submitter accounts can add data, such as new plots, accessions, phenotypic data and images. 1.2.4 Submitting Feedback on an SGN Database We appreciate your feedback! Feel free to submit any questions or suggestions by using the Feedback link provided at the footer of each page. 1.3 Menu Layout SGN Database websites have a toolbar on the top of each page with a number of menus for convenient access of major functions. The menus, as pictured below, are search, manage, analyze, and maps. The toolbar also provides a quick search, a log in button, and a new user button. The menus can be customized for each instance, so they may not appear exactly as shown on your instance; for example, most instances do not have a map menu. 1.3.1 Menu Options Search In the Search menu, the options are: Tab Description Wizard Search different accessions and plots by location, year, trial, and trait data. Can also be used to create lists of different types. Accession and plots Search accessions and plots using a variety of criteria Trials Search trials by name, description, breeding program, year, location, and trial type. Markers Search different markers Images Search images contained in the SGN database People Search database users Manage In the Manage menu, the options are: Tab Description Breeding Programs View, add and delete breeding programs Locations View, add and delete locations Accessions Manage and search different accessions Seedlots Manage and search different seedlots Crosses Create new crosses in the database Field Trials Manage field trials. Create trials using different field layouts. Genotyping Plates Manage genotyping plates. Create 96 or 384 well plates. Phenotyping Upload phenotyping files from the Tablet Field Book application Field Book App Manage the field book app data (download files to tablet) Barcodes Refers to the old barcode system, mainly historical Download Download information in the database based on lists Analyze Clicking on the Analyze link will give a full menu of all analysis functions In the Analyze menu, the options are: Tab Description Breeder Tools Breeder Home Access breeding functionalities. Lists important and helpful links. Barcode Tools Manage, create, and download barcodes. Also access barcode tools. Genomic Selection Can search for traits, start building a GS model, and predict values based on genotypes Sequence Analysis BLAST Sequence homology search Other Ontology Browser Browse all recorded ontologies 1.4 Working with Lists Lists are collections of identifiers that are stored in the database. Lists can be composed of accessions, plots, traits, locations, and trials. A given list can only contain one type of items, for example, all items have to be of type location in a list of locations. Lists are attached to the individual users account, and can only be created and seen by the user while logged in, however, lists can be made public for all other users to see. SGN databases make heavy use of lists in a number of tools on the website. For example, trials are created using lists of accessions. 1.4.1 Creating lists Lists can be generated in various ways: One way to create a list is by clicking on the Lists link located on the toolbar. To create a new list, enter the name of your new list and then click on the New List button. The name of the list can be anything, but should be unique and should be something to help you easily identify the list. If the list already exists, it will appear on the Your Lists dialog. To add items to your list, click on the View icon to open the List Contents section. On the List Contents page, enter items that you want to add to the list, then click on Add button. The page will be updated and will display your items in a table at the bottom of the dialog It is possible to sort the list if you need. Select the type of items in your list. To verify that the items that you added to your list are already stored in the database and that you selected a correct type for the items, click on the Validate button. If those items are already in the database, a message will indicate that This list passed validation Note that a list cannot contain duplicate elements. If a duplicate item is entered, the list manager will inform the user that the element is already in the list and will not add it again. Another easy way to create a list is to use the 2.1, which can be accessed from the Search menu. 1.4.2 Viewing and editing lists Lists can be viewed and edited using the Lists link on the toolbar. Clicking on the link will open a dialog that displays all of your lists, as well as an option to create new lists. This page shows all lists that have been created, including those created by using the Search Wizard. You can view and edit your lists by using Actions buttons. Clicking on the view icon will open a new window called List Contents that allows you to change the list name, the type of the list, add new items, or delete existing items. Clicking on the delete icon will delete your list. Caution: this action cannot be undone. Clicking on the download icon will download the contents of your list to your computer. Clicking on the make public icon will make your list available for other users to view and use your list. 1.5 User Permissions Breedbase accounts are assigned one or more of four different roles to determine the level of access they have within the database. The possible roles are User, Submitter, Sequencer, and Curator. Each role grants specific permissions, and careful management of them helps prevent data from being altered or deleted in error. Accounts are also assigned Breeding Program role(s) to grant access to the specfic breeding program(s) they work with. The User role gives an account permission to view and download data throughout the database. The Submitter role gives an account permission to design field experiments and to upload and edit data using the tools in the Manage section. In order to submit and manage breeding data within a given breeding program, a submitter also must have a matching Breeding Program role. The Sequencer role gives an account permission to design genotyping experiments and submit plates to a genotyping service. The Curator role gives an account permission to do all of the above, as well as to delete data within the database. The Curator role also enables the addition or deletion of roles for all database accounts in the Manage User Roles tool. "],["searching-the-database.html", "Chapter 2 Searching the Database 2.1 The Search Wizard 2.2 Accessions and Plot Search 2.3 Trials Search 2.4 Trait Search 2.5 Ontology Browser 2.6 Search Seedlots", " Chapter 2 Searching the Database You can search for information on the database by using the following search options: Wizard, which uses combined criteria specified by users; Accessions and Plots; Trials; Markers; Images; People; FAQ. 2.1 The Search Wizard 2.1.1 How the Search Wizard Works The search wizard presents a number of select boxes, which are initially empty. You start searching by picking a category of data from the dropdown above the left-most select box. Once a category has been picked, the database will retrieve all the options within this category and display them within the first select box. You then select one or more options from the first select box, which activates the second dropdown. You can then select a category from the second dropdown, and repeat this same search process through all four dropdowns and select boxes. In the example above, the locations category was chosen in the first dropdown. The first select box then displayed all the possible locations in the database. The option Ibadan was selected. This activated the second dropdown. The category years was chosen in the second dropdown. The second select box then displayed all the years that are linked in the database to the location Ibadan. From that list, the options 2011 and 2012 were selected. This activated the third dropdown. A final category, accessions, was chosen in the third dropdown. The third select box was then populated with the 3847 accessions in the database that are linked with the location Ibadan in the years 2011 or 2012. In addition to the basic search operations demonstrated above, users can take advantage of two more features: Load Selection from List Instead of picking a category in the first dropdown, users can instead populate the first selectbox from a list by scrolling down in the first dropdown to the Load Selection from List subheading and selecting a list. This is useful for starting queries with a list of plots, as this category is not among the options in the first dropdown. ANY/MIN/ALL Toggle By default, the search wizard combines options within a category using an OR query. In the example above, in the third panel the wizard retrieved accessions associated with the location Ibadan in ANY of the years 2011 OR 2012 If the user clicked the toggle below the second select box to change it to ALL before choosing accessions in the third dropdown, the wizard would instead retrieve accessions associated with the location Ibadan in the years 2011 AND 2012. This will be a smaller set of accessions, because any accessions used only in 2011, or only in 2012 will be excluded. A more advanced search could use the MIN toggle option. This allows the user to make a query in between an ANY or ALL query, where a minimum number of matches from the selected column will be used as a filter for the next column. The minimum can be provided as either a percentage (%) or an actual count of items (#). In the example above, if the years 2011, 2012, and 2013 were selected in the second column, the user could enter 2 in as the minimum and select # as the minimum match type. This would select accessions in the third column that were used in 2 or more of the selected years. 2.1.2 How to use retrieved data Getting more Info Any option in the wizard select boxes (except for years) can be clicked to open a page with more details. The new page is opened in a new tab. Saving to a list You can store the highlighted items in any selected box to lists. This is done using the inputs and buttons directly below the select box. Dont forget, you must be logged in to work with lists! To add items to an existing list, first pick an existing list using the Add to List dropdown on the left. Then click the Add button. A popup window will confirm the action, and display the number of items added to your existing list. To store items to a new list, first type a new list name in the Create New List text input on the left. Then click on the Create button. A popup window will confirm the action, and display the number of items added to your new list. Downloading Data You can download trial metadata, phenotypes and genotypes associated with the highlighted items in the wizard select boxes. This is done using the buttons in the download section at the bottom of the page. Dont forget, you must be logged in to download data! Metadata Trial metadata can be downloaded by selecting a subset of trials from the database or based on your search categories. To download, click on Related Trial Metadata, a dialog will appear. Select download format and click the Metadata button to complete your download. Phenotypes The phenotypes download is quite flexible, and can download a subset of all the trial data in the database based on whichever categories and options you currently have selected. Simply click on the Related Trial Phenotypes link, review the options, changing or adding any additional parameters you like, then click Download Phenotypes. Genotypes The genotype download is more stringent. It requires a minimum of one accession and one genotyping protocol to be selected in the wizard select boxes. The text box in the download section of the page will help track what has been selected. Once clicked, the Download Genotypes button will download a genotype file for the selected accessions. Saving the wizard selections As discussed above, the selections of the individual select boxes in the wizard can be saved separately to a list. The lists can be used as inputs in other tools on the site. However, sometimes creating a selection is quite time consuming and restoring the selections from four different lists would be cumbersome too. Therefore, the selections can be saved together in a dataset, and named for later retrieval. This is done in the section Load/Create Datasets that is below the first two wizard select boxes. To select an existing dataset, one uses the Load Dataset dropdown. A particular dataset can be chosen, and the Load button can be clicked to retrieve and display the dataset in the wizard. To create a new dataset using items that are selected in the wizard, one can enter the name of the new dataset in the Create New Dataset text box. Once the dataset has been given a name, clicking the Create button will save the new dataset. 2.1.3 Updating the Wizard The search wizard uses a copy of the database, or a cache, to return results quickly. If data appears to be missing, it usually means that the cache needs to be updated. Users with submitter privileges or above can do this using the Update Wizard button. One can also use the Refresh Lists button to update the available lists. This will take just a few seconds in small databases, but may take a few hours to complete in larger databases. 2.2 Accessions and Plot Search Accessions and their related materials (cross, plant, plot, population, tissue_sample, training population) can be searched by using Search Accessions and Plots page. On this page, accession is the default stock type; however, you can change stock type by selecting an option from the drop-down list. From this page you can construct detailed queries for stock types. For example, by using the Usage section, the Properties section, and the Phenotypes section you could search for accessions which were diploids used in a specific year and location and were also phenotyped for height. You can also search for accessions based on genetic properties, such as the location of an introgression on a specific chromosome. It is possible to query over any of the available properties, such as ploidy_level, country of origin, introgression_chromosome, etc. In the search result table it is possible to select any of the available properties to view. At the bottom of the accession search there is a phenotype graphical filtering tool. Here you can filter down accessions based on combinations of trait performance. The filtered down accessions are then able to be saved to a list. For information on adding Accessions please see the Managing Accessions help. For information on how field trial plots, plants, tissue samples, and subplots are added to the database, please see the Managing Field Trials help. 2.3 Trials Search Trials on the database can be searched based on trial name, description, breeding program, year, location, trial type, design, planting date, and harvest date. 2.4 Trait Search On the Trait Search page (menu item Search > Traits), traits in the database can be searched by ID, name, or descripiton. Optionally, a starting list of traits can be selected to filter down results. Selecting traits in the results of the search allows one to add the selected results to a trait list, or create a new trait list from the select results. 2.5 Ontology Browser A more advanced tool for searching for Traits is the ontology browser, available by clicking on Analyze and Ontology Browser. From here you can search ontologies and see the various classifications of terms in a tree display. The terms which appear in the Trait Search in 2.4 are only variable terms. The ontology browser shows these variables as different from their grouping terms by indicating VARIABLE_OF like in the following screenshot. 2.6 Search Seedlots Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. To search for available seedlots you go to Manage and then click Seed Lots. By clicking Search Seedlots, you can specify query information. The results from your search will be in the table below the search form. "],["managing-user-roles.html", "Chapter 3 Managing User Roles 3.1 What are User Roles? 3.2 The Manage User Roles page", " Chapter 3 Managing User Roles 3.1 What are User Roles? Every user account in Breedbase has one or more associated roles that determine the authorizations (what the user is allowed to do) in the database. There are three fundamental roles, curator, submitter, and user, which determine basic read/write levels. The curator status can read and write everything in the database. The submitter status can add information and edit or delete previously submitted information. The user type can only read data. Additional roles represent the breeding programs, and are sometimes used to fine-tune write and edit capabilities, as it necessary for multiple users in a breeding program to edit each others data. 3.2 The Manage User Roles page In the Manage menu, select the item User Roles. This will show the current users in the database with their associated roles. If you are logged in as a curator, the table will show system roles as well as breeding program roles; if you are logged in as a submitter or user, it will show breeding program membership. If logged in as a curator, the roles can be added or deleted. To delete a role, click on the X in the role name. A confirm dialog will be displayed to prevent accidental deletion. To add a role, click on the plus sign next to the roles. A dialog will pop up with a list of roles. Select the desired role and click Submit. The new role should be displayed next to the user immediately. Role deletions and additions will be effective immediately. It is recommended that few users be given the curator privileges to avoid confusion over data ownership and accidental data overwriting and deletion. "],["managing-breeding-programs.html", "Chapter 4 Managing Breeding Programs", " Chapter 4 Managing Breeding Programs New breeding programs can be added by using Add New Program button on the Manage Breeding Programs page. Clicking on the Add New Program button will generate a blank form for you to fill out the name and description of the breeding program that you want to add. After completing the form, click on Add Breeding Program button to finish the process. "],["managing-locations.html", "Chapter 5 Managing Locations", " Chapter 5 Managing Locations Field locations can be managed using the Manage Locations page. On this page, locations in the database are organized based on their breeding programs. Each location has a link to trials conducted in that location. To add a new location, click on the Upload New Locations button that links to the Upload Locations form. The Upload Locations describes how to build a spreadsheet with location data for upload. Name, abbreviation, country code, country name, program, type, latitute, longitude, and elevation are all required. The NOAA station ID is optional. Link a spreadhsheet to the form and click Upload to add those locations to the database. Alternatively, locations can be viewed and added via the map. Hover over an icon on the map to see the location details and trials linked to that location. Click on the map to open the new location dialog. Fill in the same information that would be used in the spreadsheet upload to add a new location. "],["managing-accessions.html", "Chapter 6 Managing Accessions 6.1 Add Accessions Using A List 6.2 Uploading Accessions and Accessions Info From A File 6.3 Email alert for accession upload 6.4 Add Parentage (Pedigree) Information to Accessions 6.5 Working with grafts 6.6 Bulk renaming of accessions", " Chapter 6 Managing Accessions The Manage Accession page provides links for adding new accessions. New accessions can be added to the database by either using a List or by uploading an Excel file (either XLS or XLSX format). Both options are explained in more detail below. To begin, click on the Add Accessions or Upload Accession Info link. This will open a dialog allowing you to select either Using Lists or Uploading a File. 6.1 Add Accessions Using A List First we will show how to add accessions Using Lists. Here you select an accession list which you have previously made (see List Manager chapter). If you need to create or edit your list you can do so now by clicking Manage Lists. After selecting your list, click Continue. The contents of the list will be checked against the database, and elements that are already present will be flagged. A dialog will appear that will show the accessions which already exist in the database. After clicking on the Continue button, the next dialog will appear with accessions that have very similar names as the accession that you are adding. In the example below, there are two accession with very similar names to accessions already in the database. TME0419 is very similar to TME419, and probably represent the same line, so it would be a mistake to add this the database again. Duplicate lines in the database should be avoided, as they cause problems when evaluating lines; data is divided up among several duplicates, making it harder to get the full picture about an accession. To avoid situations in adding a mistaken duplicate accession, the database gives you options for moving forward with these very similar looking accession names. You can either continue saving the name in your list, replace name in your list with selected existing name, remove name in your list and ignore, or add name in your list as a synonym to selected existing name. Clicking Download Fuzzy Matches will return a tabular result of the fuzzy accession name results shown. Click Make changes and continue to move on. The final dialog shows the accessions that will be added. Here you need to assign the species of these accessions. You can optionally group the accessions into a population and/or add an organization for the accessions. Once you click Add Accessions, the new accessions will be created in the database and you will see the following confirmation dialog, which includes links to the newly created accessions. 6.2 Uploading Accessions and Accessions Info From A File Uploading accessions using a file is very similar to using a list, but enables you to add a variety of attributes, such as synonyms or ploidy levels, to the accessions in bulk. Clicking on Spreadsheet format will show the required structure of the spreadsheet. The file must be XLS or XLSX format and can contain a number of header columns as attributes. It is important that you use exactly the same header column names as listed here. In columns that indicate that many attribute values can be passed at once using (s), such as synonym(s), you can pass a comma separated list of values, such as synonym1,synonym2. Once you have selected your XLS or XLSX file for upload, click Continue. The following process is the same way as with lists: The first dialog which can appear will show accession names which are already in the database. Click Continue and the next dialog that can appear will show fuzzy matches for the accession names you are trying to upload. Here you can choose to prevent adding accession names which look very similar to each other as wrongly duplicated accessions. Click Continue and the final dialog that will appear will show the information to be added into the database. Here it is divided into accession names that are new and accession names that already exist in the database; however, for the accession names that already exist it will show additional attributes that originated from your file that will be added to these accessions. Once you click Add Accessions, the new accessions and information will be created in the database and you will see the following confirmation dialog, which includes links to the created and updated accessions. 6.3 Email alert for accession upload When uploading a large number of accessions from a file, uploads can take a while, as the system needs to perform a series of checks on each entry. You have the option to receive an email notification about the status and results of your upload by clicking the Email Alert checkbox. By default, the system will use the email address associated with your account, but you have the option of entering a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results, including any warnings or errors that may have occurred during the upload. 6.4 Add Parentage (Pedigree) Information to Accessions Pedigree data can be uploaded from your computer by clicking on Upload Pedigree File You can find detailed information on how to prepare pedigree file by clicking on File format information. The file format can be tab or comma delimited text file, or Excel files (.xls or .xlsx). The currently supported format has four columns: progeny name female parent accession male parent accession type Type can be biparental, self, backcross, sib, polycross, reselected, or open. In the case of the open type, the male parent accession field can remain blank. For all other types, both columns should be filled, even if they contain the same information as another column (such as self). 6.5 Working with grafts Grafts are plants that are composed of a rootstock and a scion, which are genetically different and fused together, usually at the stem level. To work with grafts, the grafts interface needs to be activated by a system administrator. Please contact your Breedbase provider. Briefly, a configuration parameter needs to be added to the sgn_local.conf file, show_grafting_interface. It should be set to 1 in sgn_local.conf, the default is 0 in sgn.conf. Grafts to be created need to be specified using an Excel file (xlsx format) with two columns. The first column should have the header scion accession and should list accession names that will be scions. The second column should have the header rootstock accession and should list accession names that will be rootstocks. In the database, the graft accessions will be created as single accessions. The graft accession will have two relationships, one to the scion accession (scion_of relationship) and one to the rootstock (rootstock_of relationship). These relationships are displayed on the pedigree viewer. The graft accession name is created from the scion accession name and the rootstock accession name, separated by the graft separator character. By default, the graft separator character is the plus sign +. The graft separator character can be changed in the sgn_local.conf file, using the parameter graft_separator_string. The graft separator string should not occur in any other accession names that are not grafts. When the grafting interface is activated, a new button will be shown on the manage accessions page, called Upload Grafts. Clicking the button brings up the upload grafts dialog. Select the Excel file containing the grafting information. The system will validate the file, for example, check whether the accessions are in the database, and if the headers are correct. The validation result will be presented, and if problems are found, they will be listed. In addition, if there are problems, the Upload button will be grayed out and upload will not be possible. Conversely, if there are no problems, the Upload button will be activated and can be clicked to store the data. If the upload completes, a completion message is displayed with a summary what was uploaded. Grafted accessions can be used like any other accession, for example, they can be used on field layouts. If you create a list of graft accessions, use the list type accessions. Note that you shouldnt create new grafts based on other grafts. The scion accession and the rootstock accession have to be different, otherwise they will not be created. 6.6 Bulk renaming of accessions Accessions can be renamed in bulk using the rename accessions feature. To rename accessions, prepare a tab delimited file with two columns: the first column should have the header old name and contain the accession names that need to be changed. The second column should have the header new name and contain the names that the accessions in column 1 should be renamed to. The accession renaming feature is available from the Manage->Accessions page. Click on the Rename Accessions button. The first step is the upload of the file with a verification step. The verification step checks whether all the accession names in column 1 exist in the database, and whether all the accession names given in column 2 do NOT exist in the database. Only if both conditions are met, will the rename button become active, otherwise an error message is displayed listing the offending accession names. Optionally, the old name can be automatically added as a synonym to the renamed accession, using the checkbox on the submit form. This option is clicked by default. Unclick the checkbox to NOT save any old names as synonyms. Note that accession renaming should not be undertaken lightly. This feature is intended for special use cases, such as where accessions are created in a nursery with a name that is different from the accession name in the downstream breeding program. It can also be used to rename accessions in bulk that have spelling mistakes and other issues. Please note however, that the tool does not make any attempt to change the names of associated elements, such a plots, that may have been constructed using accession names. Because of the many implications of accession renaming, the feature is limited to accounts with the curator role. "],["managing-seed-lots.html", "Chapter 7 Managing Seed Lots 7.1 Add New Seedlot(s) 7.2 Seedlot Transactions 7.3 Seed Inventory 7.4 Find Seedlots For a List of Accessions 7.5 Create a seedlot for an Accession or Cross 7.6 Add quality data to a seedlot 7.7 Seedlot Maintenance Events 7.8 Deleting Seedlots", " Chapter 7 Managing Seed Lots Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. The seed in seedlots can be from crosses or can be named accessions. Seedlots from crosses would represent seed harvested. Click Manage and then Seed Lots to begin. 7.1 Add New Seedlot(s) To add a single new seedlot, click on Add Seedlot. This will bring up the following dialog where you enter information about where the seedlot exists, what accession or cross is contained in it, and how many seeds there are. A seedlot must contain either an accession or a cross, and not both. A seedlot must have a weight in grams or a seed count or both of these. In the case where you have many seedlots to add to the database, you can upload an excel XLS or XLSX file instead. Click Upload Seedlots to see the following dialog. 7.2 Seedlot Transactions Seedlots are capable of tracking where seeds came from, such as from crosses, and to where seeds go, such as to plots in the field. If you navigate to a seedlot detail page you will see the following. On this page you see and can edit information regarding a single seedlot, such as its name and location. You will also see a table indicating all t he transactions that a seedlot has been involved in, such as if it was planted in a plot in the field. Transactions to field plots are created when adding or uploading a new trial or from a trials detail page. Clicking on Add New Transaction let you add a transaction from between this seedlot and another seedlot. This kind of transaction is useful for representing if you have distributed seed to different locations. 7.3 Seed Inventory To inventory your seed: 1) Make sure your seedlots are in the database. Use Add New Seedlot to add a single seedlot or Upload New Seedlots to add many. 2) Make sure your seedlots are barcoded. You can print these barcodes from the database. 3) Use the Inventory Android Application to scan seedlot barcodes and record weight. Then use Upload Inventory to upload this info into database. If you prefer you can create your own CSV file and upload that, if you do not want to use the Inventory Application. For more info about the Inventory Android Application go to Inventory. Clicking the Upload Inventory button will bring the following dialog: The CSV file that should contain your inventory should meet these Template requirements. The Seed Inventory Android Application exports this exact file. 7.4 Find Seedlots For a List of Accessions A convenient tool for searching available seedlots for a list of accessions is available in the list tool. First open up your list of accessions. For help opening a list of accessions please see the List section help. There is a button called See Available Seedlots. Once you click this, you will see the following table in a dialog. From here you can create a list of seedlots using the checkboxes and the input at the bottom. 7.5 Create a seedlot for an Accession or Cross Complementary to what we saw above for creating seedlots from the Manage Seedlots page, it is possible to create a new seedlot from an accessions detail page or from the cross detail page. On the accession detail page, this is visible in the Related Stocks section as seen below. The cross detail page has an identical section. Notice the link for creating a new seedlot, which streamlines adding the seedlot. 7.6 Add quality data to a seedlot Quality information can be added to a seedlot in the quality field. This is also available as a column in the file upload format. It is recommended to use a controlled vocabulary, defined by the user, for the quality field. For example, good quality seed should be labelled ok, whereas other quality descriptors could be moldy, insect damage, or low sprouting, etc. 7.7 Seedlot Maintenance Events For some crops, such as sugar kelp, a seedlot requires routine maintenance for the successful long-term storage of the seedlot. (For example, a Seedlot Maintenance Event for sugar kelp would be the routine change of the water that gametophytes are kept it). Breedbase can now store a record of these Seedlot Maintenance Events associated directly with existing Seedlots. Maintenance Events can be uploaded using a simple Excel template or recorded directly on the website. 7.7.1 Setup Each Breedbase instance needs to be configured to support the storage of Seedlot Maintenance Events since each crop will have their own distinct set of maintenance events for their seedlots. To check if your Breedbase instance supports this feature, go to the Manage menu and select the Seed Lots page. Make sure you are logged in and look for the Seedlot Maintenance button near the top, next to the Create Seedlot(s) and Upload Inventory buttons. If you dont see this button, contact the developer(s) supporting your Breedbase instance and ask if they can setup this feature. The location of the Seedlot Maintenance button on the Manage > Seed Lots page 7.7.2 Adding Events Seedlot Maintenance Events can be added using two methods: 1) Uploading an Excel template or 2) Recording events directly on the website Uploading Events with Excel Template To bulk-upload a file of Seedlot Maintenance Events, first create an Excel (.xls or .xlsx) file with the following headers: seedlot - the name of the Seedlot to associate the event with (must exactly match an existing Seedlot in the database) type - the name of the Seedlot Maintenance Event type (these vary between Breedbase instances, a list of supported event types is displayed on the upload page) value - the value of the Seedlot Maintenance Event (these may be different for each event type and vary between Breedbase instances, a list of supported event values is displayed on the upload page) notes - optional, additional notes/comments about the event operator - the username of the Breedbase user that recorded the event timestamp - the date/time the event was recorded, in YYYY-MM-DD HH:MM:SS format Once you have an Excel file with the events filled out, follow these steps to upload the events to the database: Make sure you are logged in to your Breedbase instance Go to the Manage > Seed Lots page Select the Seedlot Maintenance button Select the Upload Maintenance button Choose your Excel (.xls or .xlsx) file to upload Select the Upload button The Seedlot Maintenance upload dialog, showing the supported event types and values (for sugar kelp) Recording Events on Website To add individual Seedlot Maintenance Events to the database in real time, as theyre being recorded, use the Record Maintenance page. Follow these steps to record Seedlot Maintenance Events: Make sure you are logged in to your Breedbase instance Go to the Manage > Seed Lots page Select the Seedlot Maintenance button Select the Record Maintenance button Enter the Seedlot Name or scan a barcode that has the Seedlot Name encoded. Once entered, the box at the top of the page will display basic information about the Seedlot as well its recently recorded events. Select or Enter the values of individual events Optionally, notes button next to each event to add additional notes/comments about that specific event Make sure the operator/username and timestamp are correct Select the Submit button to add the recorded events to the database. NOTE: any events that remain selected as Not Recorded will not be submitted to the database. The Seedlot Maintenance record page, as configured for sugar kelp 7.7.3 Displaying Events Recently recorded Seedlot Maintenance Events are displayed in a table from the main Seedlot Maintenance page, as well as the detail page for individual Seedlots. Unfiltered table of recent Seedlot Maintenance events The events displayed in these tables are sorted by timestamp, with the most recently recorded events displayed first. The displayed events can be filtered using any number of supported filter criteria, such as: - seedlot names (as entered on the page or using an existing seedlot list), - dates (on, on or before, before, on or after, and/or after the entered dates) - event types - event type values - operator/username Select the properties of the filter(s) you want to apply, then select the Add button next to the button to add the filter to the list of applied filters. Once youre done adding filters, select the Filter button to search the database for the filtered events. A filtered table of Seedlot Maintenance events The filtered events can be downloaded directly from the table using the Excel or CSV buttons at the top of the table. Or Seedlot Maintenance Events can be bulk-downloaded (this includes all events for a Seedlot) using a list of Seedlots from the main downloads page (see below). 7.7.4 Downloading Events To bulk-download all events for a specific subset of Seedlots: Create a list containing the Seelots you are interested in. Go to the Download Using Lists page (Manage > Download) Find the Download Seedlot Maintenance Events section Select your list of Seedlots Select the Download button to generate the download file The downloaded file will follow the same format as the upload template and will contain all recorded Seedlot Maintenance Events for each Seedlot in the list. 7.8 Deleting Seedlots Seedlots can be deleted on the Manage Seedlots page (/breeders/seedlots) by search the seedlot and then clicking the X to delete one seedlot at a time. To delete a seedlot, the logged in user needs the required delete privileges on the seedlot. The seedlot also should not have any transactions associated with it (except for the initial transaction). To delete seedlots in bulk, generate a list of type seedlot, for example, using the wizard. Open the section Delete seedlots using a list on the Manage Seedlots page and select the list. Seedlot deletion using a list is only available to user with curator status. "],["managing-populations.html", "Chapter 8 Managing Populations", " Chapter 8 Managing Populations Populations are modeled as groups of accessions. This grouping can be useful in downstream analyses. To manage these populations go to Manage Accessions and scroll tp the bottom. To add a new population click Create Population. The following dialog will appear where you choose a list of accessions and give a name to the new population. Please note it is also possible to create a population when you are uploading new accessions into the database. Click on the plus (+) button next to Populations to see all the available populations. Click on a population name to see the accessions in the population. From here you can delete accessions from a population as well as add new accessions to the population. "],["managing-crosses.html", "Chapter 9 Managing Crosses 9.1 Crossing Experiment 9.2 Cross 9.3 Cross Wishlist 9.4 Crossing Experiment Detail Page 9.5 Cross Detail Page", " Chapter 9 Managing Crosses Information for crosses can be managed using the Crosses option in the Manage menu. 9.1 Crossing Experiment Different crosses in the same trial/nursery/project are grouped via crossing experiment. Crossing experiments are organized based on their breeding programs. To find a crossing experiment, you can either type the crossing experiment name in the Search box, or look for the crossing experiment directly in its breeding program by clicking on the + icon. In each breeding program, crossing experiments can be placed directly in the breeding program, or organized in folders. The Folders section allows you to place crossing experiments in folders, move a crossing experiment in a folder to another folder, or rearrange your folders within a breeding program. 9.1.1 Add New Crossing Experiment To add a new crossing experiment, click on Add Crossing Experiment link. Required Information: Crossing Experiment Name: enter a name for the crossing experiment. The crossing experiment name must not already exist in the database. Breeding program: select a breeding program that is available in the database. New breeding programs can be added on the Breeding program page, accessible from the Manage menu. Breeding Program Page Location: select a location for the crossing experiment. New locations can be entered on the Locations page, accessible from the Manage menu. Location Page Year: select a year. Description: enter a description for the crossing experiment. After filling in the information, click Submit to generate the crossing experiment. 9.2 Cross 9.2.1 Add New Crosses Add a cross by using the Add New Cross dialog To add a single new cross, click on Add Cross link. Enter cross information in the popup dialog. Required Information: Crossing experiment: select a crossing experiment available in the database. Location: select a location available in the database. Cross name: enter a name for the cross. The cross name must not already exist in the database. Cross type: the options for cross types are: biparental, self, open pollinated, bulk, bulk selfed, bulk and open pollinated, double haploid, polycross, reciprocal and multicross. The Female Parent and Male Parent field are auto-complete fields for accessions that are already in the database. The parents specified will be entered in the pedigree of the new accessions generated by this cross. Optional Information: Female Plot and/or Male Plot: In addition to the accession names, specific plots used in the cross can also be added to the database. To retrieve plot names associated with each female/male accession, enter your trial name, then click Search Plots. Plot names of each parental accession in that field trial will be shown in the drop-down list, you can then select the plot used in the cross. Additional crossing experimental information such as pollination date, number of flowers, number of fruits, number of seeds can be specified during adding new cross. Alternatively, this information can be updated or edited directly on the Cross Details page. If you know the number of accessions that are generated from the cross, they can be instantiated immediately in the database by clicking the Add accessions for progeny checkbox and specifying the number. Click Submit to generate the cross. Upload New Crosses To upload new crosses from an Excel file (.xls or .xlsx), click on Upload Crosses link. Select a crossing experiment and a location available in the database from drop-down lists and choose a file that you want to upload, then click Upload File. Please check spreadsheet format carefully. The file must be an Excel file (.xls or .xlsx). 9.2.2 Update Crosses by Uploading To upload progenies and/or experimental info of crosses already in the database, go to Manage-Upload page. In the Crosses section, there are links for uploading progenies and experimental info. Please check spreadsheet format in each link carefully. The file must be an Excel file (.xls or .xlsx). Note: crossing experimental information is customized based on the need for each crop. As a result, column headers for experimental info in your database may be different from the information shown in this manual. 9.3 Cross Wishlist An Android ODK application is being developed to record cross information on a mobile device in the field. To link this mobile application with the database, the Cross Wishlist can be used to create a plan for which crosses to perform. This tool is available on the Manage Cross page. It is currently only available on certain databases, so when you click this link you may see an alert mentioning that the cross wishlist is not available on your database. 9.3.1 Create a Cross Wishlist Step 1. Select the accessions to be crossed in your trial There are two interfaces for this step, either Not Using Lists or Using Lists. Depending on if you already have a list of female and male accessions to use, you can decide on which interface to use. The end result of using either interface is the same. We will start by showing Not Using Lists. First select the trial in which the crosses are to be performed. This will populate a select box with all the accessions used in that trial. From here, one or many accessions can be selected as the female accession. Once the female accessions are selected, a table is populated. Each row in this table begins with the female accession that was selected, followed by a select box with all the accessions used in the trial. From here, one or many accessions can be selected as the male to use in the cross. Once the male accessions are selected to cross with each female accession, a table indicating priorities appears. Priority is meant to indicate an order in which to attempt the cross; first the highest priority male will be considered, but if this cross is not possible then subsequent males will be considered. An equal priority can be given and this will not indicate a specific order to follow. Alternatively, we could have used the Using List interface instead. Here we select the trial in which the crosses will be performed and we provide a list of accessions to consider for the females and the males to be crossed. Step 2. Select the female plots to be considered in the crosses After selecting your lists, the table below is populated. The first column has all the female accessions specified and the header row has all the male accessions specified. The males to consider crossing with each female are indicated with priority. After female and male accessions are selected to cross, either by the Nor Using List or Using List interface, click Next. The next dialog will allow selection of specific female plots to use for the cross. Sections for each female accession selected will appear with the field layout displayed. Selecting all plots in which the female is present indicates that the cross should be performed on all plots where that female accession is present. Step 3. Transfer the cross wishlist to your mobile crossing application Clicking Push Cross Wishlst for ODK Use will send the cross wishlist plan to the ONA server for use by the mobile ODK application. Crosses can then be performed and recorded in the field using the mobile application. Afterwards, the crosses are sent back to our database and stored. 9.4 Crossing Experiment Detail Page Information for crosses in the same crossing experiment is compiled in the crossing experiment detail page. Each cross name, female parent, male parent, female plot and male plot has a link to its own detail page, which contains information specific to each one. Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. 9.5 Cross Detail Page Information of each cross can also be viewed in its detail page. This page allows you to update or edit crossing experimental information and add progenies related to that cross. Note: crossing experimental information is customized based on the need for each crop. As a result, the details of the information in your database may be different from the information shown in this manual. "],["managing-field-trials.html", "Chapter 10 Managing Field Trials 10.1 Trial Detail Page 10.2 Adding Trials 10.3 Updating Trial Data 10.4 Deleting Trial Data", " Chapter 10 Managing Field Trials To view trial details on the database, click on the Field Trials link under the manage menu on the toolbar. Clicking on the Field Trials link will bring you to the Manage Trials page. On this page, trials are organized according to their breeding programs. To access trial details, click on the + icon next to your breeding program. Trials can be placed directly in their breeding program. Alternatively, they can be organized by using folders within each breeding program. Clicking on trial name will take you directly to the trial details page. 10.1 Trial Detail Page The trial detail page displays important information about individual trials including breeding program, location, year, description of the trial, design, and any files associated with that trial. Below the trial details you will find various menus for accessing and modifying trial data. There are sections for printing labels for your plots or plants, recording phenotypes, viewing your trial layout or design, viewing phenotypes for this trial, or conducting analyses. The transplanting date field feature will only be displayed if it has a value. To add a transplanting date after creating a trial, change the show_transplanting_date parameter from 0 to 1 in the SGN config file. As a result, you will be able to add a date under the transplanting date field by clicking the Edit Trial Details on the trial detail page. 10.2 Adding Trials Only users with the account status of submitter may create trials. To learn how to change your account status from user to submitter visit section 1.2. 10.2.1 Prerequisites To add a trial, all of your accessions should already exist in the database before you begin to design a trial. If you have accessions that are not in the database, see the instructions in Managing Accessions. The breeding program and location for your trial should also exist in the database. If you need to add breeding program and/or location to the database, see the instructions in Managing Breeding Programs and Managing Locations respectively. On the Manage Trials page, there are two methods to create trials: by selecting Upload Existing Trial(s) to create a trial or trials from a spreadsheet; or by selecting Design New Trial and entering the trial data by hand. 10.2.2 Adding a trial by using Design New Trial form Step 1. Begin the Design New Trial workflow Click on Design New Trial to begin. The first step in this workflow is an introduction that looks like this: Here it gives information about what is required for a trial, including that to create a new trial, you need to create a list of the accessions that you would like to use in the trial. Lists can be viewed, created, and modified with the lists tool at the upper right of the screen. For more information on lists, click here. Step 2. Enter Trial Information On this screen you need to enter basic information about the trial, such as breeding program and location(s). You must also select a design type, such as Complete Block Design. The design is important because it influences how your genotypes are distributed and randomized over the trial. You must first click validate before proceeding to the next step. Step 3. Enter Design Information On this screen you need to specify a list of accessions to use in the experiment. This list must be a valid list of accessions. You must also specify all required design information, such as number of replicates. In this case, the number of blocks must be given. Step 4. Enter Trial Linkage Information (Optional) This next sections allows you to associate this new trial with other field trials, crossing experiments, or genotyping plates already present in the database. This is optional, and can be completed at a later date from the trial detail page. Step 5. Enter Field Map Information (Optional) On this screen you can specify how the row and column numbers will be generated for the plots in the trial. The row and column number represent a relative position of the plot in the field. If you are not exactly sure of how you will plant the plots in the field or you have an irregular (non-rectangular) layout, you can skip this step for now. This information can be added on the Trial Detail Page once the trial is saved in the database in order to reflect exactly how the plots were planted in the field. Step 6. Custom Plot Naming (Optional) On this screen it is possible to change the format in which plot names will be generated for your trial. It is recommended to skip this step and just use the format generated by the database by default. Step 7. Review Designed Trial On this screen you can review the trial that the database has generated. You will see a graphical representation of the trial. The numbers on the squares represent the plot_number of each plot and on mouse hover you can see further information about the plot. You will also see a table representation of all the plots and their information. If you want to redo the randomization, you can click the Redo Randomization button. At the bottom there is a brief summary of the trial followed by two buttons. Step 8. Add Field Management Factors to your design (Optional) You can add Field Management Factors by clicking Add Field Management Factor(s) to Design. Clicking this opens a dialog to specify your factor. You can name this to account for fertilizer, watering regime, inoculation, or anything else. The types of management factors available is customizable in the SGN configuration file. This is optional and can be added from the trial detail page afterwards. Click Continue and a dialog will appear where you can specify plots or plants (if you added plants during trial creation) for which the factor was applied. There is a select all button as well. Step 9. Saving new trial in the database Once you are done reviewing the trial you can click Confirm to save the generated trial into the database. Once the trial has saved you will see the final completion screen: 10.2.3 Adding a trial from an uploaded file If you already have trial design layout in a spreadsheet, you can add your trial into the database by using the Upload Existing Trial(s) button on the Manage Trials page. Enter the information in the workflow to upload a trial from a spreadsheet. Step 1: The first step is to understand what the format of the trial upload is. It is important to understand that the field layout represents plots in the experiment. Each plot has a globally unique plot_name, a sequential plot_number that is unique in the trial (but not globally unique. e.g.101, 102, 103 for three separate plots), an accession_name representing what genotype is planted in that plot, and a block_number representing design replication. Each plot can be thought of as having a row_number and a column_number representing the relative position of the plot in a grid (e.g.the top left plot is row 1 column 1 following by row 1 column 2). Each plot can be planted with an amount of seed from a seedlot, where the seedlot_name represents the specific seed packet that was used, and num_seed_per_plot and weight_gram_seed_per_plot represent amount that were transferred from the seedlot_name to the plot_name. Treatments can be applied onto plots using additional column names in your file, where a 1 represents if the treatment was applied to the plot and an empty cell means it was not applied. The following page will allow you to pick a file for upload, including uploading multiple trials at once. On this page you can also inspect the file requirements for both single trial and multi-trial uploads. Minimum File requirements All accession names in the file must exist in the database. See adding accessions for more information. The uploaded file can be excel (.XLXS or .XLS), comma-separated values (.CSV), tab-separated values (.TSV) or semicolon-separated values (.SSV). The first row (header) must contain the column names: plot_name accession_name plot_number block_number is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number Only accession_name, plot_number, and block_number are required. Minimal Example: plot_name accession_name plot_number block_number is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number 2018plot1 my_accession1 101 1 1 2018plot2 my_accession2 201 2 2018plot3 my_accession2 102 1 2018plot4 my_accession1 202 2 1 When uploading multiple trials, the requirements are the same, but with additional headers to differentiate trials in the same file: trial_name breeding_program location year transplanting_date design_type description trial_type trial_stock_type plot_width plot_length field_size planting_date harvest_date. Additionally, when uploading multiple trials, you may choose to get an email when the upload is complete (see section ?? ) File validation In case of errors in the uploaded file such as missing or invalid data, a window will appear listing the specific errors in the file that must be corrected before a successful upload. Uploading a trial with Treatments You can upload a trial with treatments by adding additional column(s). The column header will be the treatment e.g.fertilizer, watering regime, inoculation, etc. and the values in these columns will be either 1 or empty, indicating that the treatment was applied to the plot or not. Step 2: Once you feel that your experiment field layout is in the right format, click on to the Next Step. You will see the following form which must be filled in completely: The trial name must be globally unique in the database. Please try to follow standard naming conventions for your group. Step 3: Go to the next page where you can link this trial to other projects. Validate the form, and then you can click Upload Trial. Step 4: In the case where you have uploaded an experiment using accession_names that are not already present in the database, you will be taken to this screen. If the accession_names in your file are all already in the database, this step will be skipped. The reason it is necessary for your accessions to be in the database before you can add a trial using them is that a single accession can be used among many trials and therefore must exist as a separate entity in the database; because of this it is also very important to be careful about adding wrongly duplicated accession_names into the database. From this screen it is possible to make a new list with the missing accession_names and then click Add Accessions to the database to immediately resolve the issue. Once all your accessions are in the database, click to move to the Next Step. Step 5: In the case where you have uploaded an experiment using seedlot_names that are not already present in the database, you will be taken to the next screen. If the seedlots in your file are all already in the database, this step will be skipped. The reason it is necessary for your seedlots to be in the database before you can add a trial using them is that a single seedlot can be used among many trials and therefore must exist as a separate entity in the database. Once all your seedlots are in the database, click to move to the next step. Step 6: If there are any other errors with your file, such as if the plot_names are not globally unique in the database or your plot_numbers are not unique in your trial or row_number is not an integer or any other error, you will see the errors listed in the red box. It is up to you to correct these errors in your file. Simply open up the file you selected earlier in Excel and correct the issues and then save the file. Then you can click Submit Trial and it will resubmit it for you. You can continue to edit your file here and submit as many times as you need until it is accepted. Completion screen Whether you were lucky enough to submit your trial successfully on Step 2 or if you tried many times on Step 5, once your trial has been saved in the database you will see the following screen: 10.2.4 Multi-location trials To add multi-location trials, simply select the multiple locations while using the Add Trial form. This will create a separate trial for each selected location, but they will share the same design and will be grouped in a single folder. By default each trial design will have a fresh randomization, but if desired you may check the Use same randomization for all locations option. Email alert for multiple trial design upload When uploading multiple trials from a file, you have the option to receive email notifications by clicking the Email Alert checkbox. By default, the system will use the email address associated with your account, but you have the option to enter a different email address if you prefer. After submitting, the upload process runs in the background, allowing you to continue using the interface without interruptions. Once the process completes, you will receive an email with the upload results. 10.2.5 Viewing Plot Layout and Trait HeatMap 10.2.5.1 Viewing plot layout In the Field Layout Tools and Phenotype Heatmap section of a Trial Detail page, the trial physical layout is displayed by default. The relative position of the plots will be displayed based on the row and column positions given to the plots during the trial creation or upload steps. The plots are color-coded based on the plots rep and block numbers and whether or not it is used as a check. Hover the mouse over the plot to see details about a specific plot. 10.2.5.2 Viewing plot layout for multiple trials If there is more than one trial grown in the same physical field, the trial layouts of all of the trials can be shown together if the trials share these properties: Each trial has the same year Each trial has the same location The location type of the trials location is set to Field The row and column positions of all of the plots (across the related trials) dont overlap. For example, trial #1 starts at row 1 and trial #2 starts at row 10. When these conditions are met and you check the Select Trials in Same Field checkbox, the plots from all of the related trials will be displayed on the same field layout. The plots will be color-coded by trial. The planting order and harvest order downloads will include the plots from all of the displayed trials in the order in which the plots occur in the field. Tracking plot images on fieldMap Plot images can be seen on fieldMap if a plot is associated to any image. To view plot image(s), click on a plot, a dialog will appear. In this dialog you will see a detailed overview of the plot, including what stock(s) are contained within it and their layout in the plot, if applicable. At the bottom of the dialog, you will see options to change the plot name or accession as well as view plot images. On the resulting dialog, click on View Plot Images. To see more images if a plot has more that 2 images, click on See more images Medium size of an image can be viewed by clicking on an image. Viewing assayed trait heatmap The phenotype heatmap can be viewed by selecting a specific assayed trait from the selectbox drop-down. Mousing over the plots highlights the plot in green and also displays the plots field information including the selected traits phenotype value. Suppressing Plot Phenotype Clicking on a plot on the heatmap would display a dialog that has a button for suppressing a plot phenotype value for a given trait. A suppressed plot value can be excluded during trial analysis and phenotype download. 10.2.5.3 Correcting spatial autocorrelation For trials with spatial layout information and stored phenotypes, you can check and correct for spatial autocorrelation by clicking the Calculate Spatial Correction button above the field map. Doing so will open the spatial corrections dialog. For an in-depth review, visit the chapter on data analysis tools. 10.2.6 Adding additional information in the Trial Detail page After you added a new trial to the database, you can edit trial details or add more information for that trial through the Trial Detail page. #### Uploading Physical Trial Layout {-} You can upload physical trial layout by clicking on the Upload Spatial Layout button on the Trial Detail page. Please check file format carefully. You can find file format information by clicking on the Spreadsheet format on the Upload Spatial Layout window. Spreadsheet format: Physical Trial Layout File requirements All plot names in the file must exist in the database. The uploaded file should be tab delimited (txt). The first row (header) must contain the column names Example: plot_name row_number col_number plot1 1 1 plot2 1 2 plot3 1 3 Select the trial layout coordinates file that you want to upload for this trial, then click OK button to upload the file. The following message is displayed after the coordinates are uploaded. Downloading Field Map Spreadsheet Field map spreadsheet can be downloaded if the trial has field coordinate (row and column numbers) uploaded for it plots. To download, click on the Download FieldMap Layout link on the Trial Heatmap section. Editing Physical Trial Layout The Usage Help button contains information on how to edit physical trial layout. There are two different options for editing trial layout: Replacing plot accession by clicking on the plot in the layout. Replacing trial accession by using the Edit Field Layout button. To edit a specific plot, click on that plot. Enter a new accession on the Edit Plot Info form, then click the Replace Plot Accession button. To replace an accession (in every plot/plant of that accession), click on the Edit Field Layout button. On the Edit Field Layout window, click the Replace Accession button. Select any accession that you want to replace, enter your new accession, then click the Replace Trial Accession button. ### Downloading the Trial Layout from the Trial Detail page Click on Download Layout on the Trial Detail page under Experimental Design. The trial layout includes all information regarding the observation units in the experiment. The observation units can be plots, plants, or subplots. The trial layout can include trial design information such as the block_number and rep_number. It can also include physical map information such as the row_number and col_number, if that information is available for the trial. The trial layout also includes information regarding treatments that have been applied in the field. Optionally, the layout can give information regarding accessions global performance for a list of traits. 10.2.7 Adding Plant Entries To Your Trial After you added a new trial to the database you can choose to add plant entries to your trial. Adding plant entries enables plant level phenotyping. It is generally better to enter data at the plant level into the database because it is always possible to calculate plot level phenotypes from the individual plant data. Plant entries can be added to your trial in four ways: (1) Automatically generated by the database. The only input required is the number of plants per plot. (2) Uploaded in an XLS or XLSX file. This allows you to specifically name your plant entries. (3) Uploaded using plant index number instead of name (the name is automatically generated) (4) Uploaded using the number of plants per plot. This allows different numbers of plant per plot, as opposed to option 1. Additionally, if a trial has had subplots added, each of these options can be used to add plants to subplots. Note that once subplots are added, plants cannot be added directly to plots, and once plants have been added, subplots cannot be created. These options are available in the Plant Entries section on the Trial Detail page under Experimental Design, as shown below. Automatically Generate Plant Entries Clicking on Add plant entries opens the following dialog box. The only input required is the number of plants per plot. This will create plant entries that are named as a concatenation of the plot_name and the plants index number e.g.plot_name_plant_1. You may optionally add row and column data to give plants a spatial layout within each plot. Upload Plant Entries Alternatively, you can choose to upload an XLS or XLSX file that contains the names of the plant entries, the plant index numbers, or the number of plants per plot. Each option comes with a dialog that specifies the file formats. 10.2.8 Adding Tissue Sample Entries To Your Trial Some trials require tissue samples to be collected from plants in a field trial. The database will generate these tissue sample identifiers for you and will maintain all relationships with the plant, plot, accession, etc. To begin, go to the Design section of a trials detail page and open the tissue sample entries section. Please note that tissue samples are directly related to plants, therefore your trial requires plants before you can add tissue samples. When you click on Add tissue sample entries you will see a dialog where you specify the number of tissue samples you require per plant. Once you have specified how many tissues samples, you can give specific words to distinguish samples, such as root or stem, as seen below. Once you have added tissue sample entries they will appear in the design section of the trial as seen below. Each tissue sample has a detail page where you can add information about the sample, such as if it is in transit or in storage somewhere. The related stocks section near the bottom of this detail page displays the relationships between all stocks, including tissue samples. 10.2.9 Uploading GPS Coordinates For Plots You can upload GPS coordinates for the plots in your trial. There is a link on the Trial Detail Page as shown below. Clicking on this link will bring up the following dialog. Here you can upload an XLS or XLSX file. To see information on the format of the file that should be uploaded, click on Spreadsheet format. This will bring up the following dialog. This dialog tells you that the file must be XLS or XLSX and must contain: plot_name WGS84_bottom_left_x WGS84_bottom_left_y WGS84_bottom_right_x WGS84_bottom_right_y WGS84_top_right_x WGS84_top_right_y WGS84_top_left_x WGS84_top_left_y The GPS coordinates should be WGS84 format and specify a four-pointed polygon around the plot. 10.2.10 Repetitive Measurements Section If a trial includes repetitive traits or time-series values, you can effectively view and analyze these values through the Repetitive Measurements Section. Start by selecting the desired trait from the trait drop-down menu. Next, define the date range by either using the date-range picker or an interactive slider, which allows you to dynamically adjust the period you wish to examine. Once the date range is set, determine how to handle the repetitive measurements by choosing from various options such as First Value, Last Value, Averaged Value, Sum Values, or All Values. Choosing the All Values option enables an additional feature that visualizes the trend of the values over time, helping you identify patterns and trends within the data. 10.2.11 Uploading Additional Files To Trial It may be of interest to you to upload additional documents, images, or recordings to your trial. To do this, scroll down to the Uploaded Additional File section on the trial detail page. From here you can view and download any of these additional files. To upload an additional file, click on the Upload Additional Files link. A dialog will appear where you simply select your desired file. For information, you can click Upload information to see the following message. 10.3 Updating Trial Data To updated the trial-level metadata (such as the planting date, design type, description, etc) of one or more existing trials, click the Update Existing Trial(s) button from the Manage > Field Trials page. This upload can also be used to rename trials or move trials to a different breeding program. In order to update a trial, you must be a curator or a submitter (that is associated with the breeding program of the trials). Here you can upload a file that contains the new metadata for the existing trials in the database. The first column is labeled trial_name and includes the name of the existing trial. Additional columns can be included for the metadata you want to update. Any columns not included in the file or values left blank will leave the existing metadata unchanged. The columns that can be included are: new_trial_name: A new name for the trial, must not already exist in the database breeding_program: The name of breeding program that managed the trial, must exist in the database. location: The name or abbreviation of the location where the trial was held, must exist in the database. year: The year the trial was held. transplanting_date: The transplanting_date of the trial was conducted. Date in YYYY-MM-DD format or remove to remove the date planting_date: Date of Planting in YYYY-MM-DD format or remove to remove the date harvest_date: Date of Harvest in YYYY-MM-DD format or remove to remove the date design_type: The shorthand for the design type, must exist in the database. Possible values include CRD: Completely Randomized Design, RCBD: Randomized Complete Block Design, RRC: Resolvable Row-Column, DRRC: Doubly-Resolvable Row-Column, ARC: Augmented Row-Column, Alpha: Alpha Lattice Design, Lattice: Lattice Design, Augmented: Augmented Design, MAD: Modified Augmented Design, greenhouse: undesigned Nursery/Greenhouse, splitplot: Split Plot, p-rep: Partially Replicated, Westcott: Westcott Design description: Additional text with any other relevant information about the trial. trial_type: The name of the trial type, must exist in the database. Possible values include Seedling Nursery, phenotyping_trial, Advanced Yield Trial, Preliminary Yield Trial, Uniform Yield Trial, Variety Release Trial, Clonal Evaluation, genetic_gain_trial, storage_trial, heterosis_trial, health_status_trial, grafting_trial, Screen House, Seed Multiplication, crossing_block_trial, Specialty Trial plot_width: plot width in meters plot_length: plot length in meters field_size: field size in hectares 10.4 Deleting Trial Data To delete a trial data, click on the Delete trial data section. There are links to delete traits, layout and trial entry data. To delete assayed trait data, click on Delete trait data link. On the appeared dialog, confirm deletion by clicking on the Select Traits For Deletion button, then select one or more traits to delete from the trial. To delete trial layout data, click on the Delete layout data link. Confirm deletion on the appeared dialog. To Delete trial entry, click on Delete trial entry link. Confirm deletion on the appeared dialog. "],["managing-genotyping-plates.html", "Chapter 11 Managing Genotyping Plates 11.1 Adding a New Genotyping Plate 11.2 Genotyping Plate Detail Page", " Chapter 11 Managing Genotyping Plates Genotyping Plates represent the content of a genotyping plate sent to a genotyping facility (e.g.samples in specific wells). To streamline this process, it is possible to upload this information or let the database create a plate for you. Once the genotyping plate is saved in the database it is then possible to export the information directly to genotyping facilities that are BrAPI compliant. The genotyping facility can then provide status information to us via BrAPI. To begin go to Manage->Genotyping Plates. Here the genotyping plates are divided by Breeding Program. These sections can be expanded by clicking on one. 11.1 Adding a New Genotyping Plate To begin, click on Add Genotyping Plate. Notice that this form is split into three sections: Plate Information, Well Information, and Confirm. The first section is for defining information about the genotyping plate, such as a Plate identifier, plate format (96 well), etc. The second section is for defining the samples in the wells, such as sample names, sample concentrations, well position, etc. The final section is for Submitting the info. All fields in the Plate Information section are required. In the Well Information section you can choose to either 1) Upload an XLS or XLSX spreadsheet with your sample layout or 2) let the database create the sample layout. If you choose to upload an XLS or XLSX spreadsheet, the Spreadsheet Template info requires the following: In either case, the sample identifier is generally a concatenation of Plate name and well position, e.g.MyGenotypingTrial1_A01. In either case, you need to provide a source_observation_unit_name for each sample. This can be a tissue sample name, a plant name, a plot name, or an accession name; however, in any case, the identifier must already exist in the database. This allows us to link the sample in the well to specific field trial plots, or, plants, or tissue_samples. If you only know which accession is in the well, you can use the accession name. In the final Confirm section you can decide whether to submit this information to the genotyping facility you selected. This requires that the genotyping facility is BrAPI compliant to work. 11.2 Genotyping Plate Detail Page If you open a specific genotyping plate, it will take you to the detail page. Here you can see the Accessions used in the plate (if you created the trial and the source_observation_unit_names you used were plots, this will still work because we know the accession of the plot or plant or tissue sample). Further down you can see a graphical representation of your plate with well positions. This can be 96 well or 384 well depending on your plate format. "],["Using-Field-Book-with-BreedBase.html", "Chapter 12 Using Field Book and BreedBase for Data Collection and Storage 12.1 Introduction 12.2 Installing Field Book 12.3 Preparing Field Layout Files in BreedBase 12.4 Importing Field Layout Files into Field Book 12.5 Adding Traits to Field Book Using BreedBase 12.6 Collecting and Exporting Phenotype Data 12.7 Using BrAPI for Data Transfer Between Field Book and BreedBase", " Chapter 12 Using Field Book and BreedBase for Data Collection and Storage 12.1 Introduction Field Book is an open-source Android application designed to streamline phenotypic data collection, replacing traditional paper field books and reducing transcription errors. BreedBase supports integration with Field Book by enabling users to generate properly formatted field layout and trait files, import collected data, and optionally exchange data via BrAPI. This section provides step-by-step guidance on preparing Field Book for data collection using trial data stored in BreedBase, including file generation, manual file import, and data export. It assumes that trial fields and trait definitions already exist in BreedBase and that users have access to an Android device. Field Book User Manual 12.2 Installing Field Book Field Book can be installed on an Android device via the Google Play Store or manually from GitHub. To ensure access to the latest features and fixes, we recommend checking for updates regularly. 12.2.1 Option 1: Downloading from the Google Play Store Download the Field Book app directly from the Google Play Store or scan the QR code below. Walkthrough: Installation and Setup 12.2.2 Option 2: Manual Installation from Github Walkthrough: Installing Field Book from GitHub 12.2.3 Keeping Field Book Updated Regular updates are recommended to access new features and bug fixes. Walkthrough: Updating the Field Book App. 12.3 Preparing Field Layout Files in BreedBase Field Book requires a field layout file containing a unique identifier for each plot or plant to be observed. Optionally, columns for row, column, pedigree, or previous phenotype data may be included for context. BreedBase provides tools to generate layout files directly from trial metadata. These files can be exported and then uploaded to Field Book for data collection. Walkthrough: Creating a Field Layout File in BreedBase 12.4 Importing Field Layout Files into Field Book Field layout files can be imported into Field Book via cloud storage or directly from the device. 12.4.1 Option 1: Import from Cloud Storage Requirements: A generated field layout file. Linked cloud service (e.g., Google Drive) on the device. File uploaded to the cloud account. Walkthrough: Importing from Cloud Storage 12.4.2 Option 2: Import from Local Storage Device Requirements: A generated field layout file. File transferred to the Android devices local storage. Walkthrough: Importing from Local Device 12.5 Adding Traits to Field Book Using BreedBase For exported data to be compatible with BreedBase, trait names in Field Book must match the exact names stored in the database. BreedBase supports generation of trait files containing properly formatted trait names, types, units, and metadata. 12.5.1 Creating Trait Files in BreedBase Walkthrough: Creating Trait Files in BreedBase 12.5.2 Importing Trait Files to Field Book Walkthrough: Importing Trait Files 12.6 Collecting and Exporting Phenotype Data This section describes the process for recording and exporting phenotype data using the Field Book application. It assumes that the required field layout and trait files have already been imported into the app. Users will be guided through entering phenotypic observations in the field and exporting collected data in a format compatible with downstream processing and upload to BreedBase. This standardized workflow helps ensure data integrity, traceability, and seamless integration with breeding databases. 12.6.1 Collecting Phenotype Data in Field Book Once field layout and trait files are loaded, data collection can begin. Field Book allows users to view each plot or plant entry, record trait values in structured forms, and automatically timestamp and geotag observations (if device settings allow). Walkthrough: Collecting Data in Field Book 12.6.2 Exporting Phenotype Data from Field Book and Importing to BreedBase After data collection is complete, Field Book allows users to export data as a .csv file. This file can then be uploaded to BreedBase for storage and analysis. Walkthrough: Exporting Phenotype Data from Field Book Once exported, the .csv file can be imported into BreedBase through the trials data upload interface. Proceed with uploading phenotype observations as described in the Managing Phenotypic Data section. It is important that: Trait names match those defined in the database. Plot or plant identifiers align with the original layout. 12.7 Using BrAPI for Data Transfer Between Field Book and BreedBase Field Book supports integration with BreedBase via the BrAPI (Breeding API) specification. BrAPI enables direct exchange of trial metadata and phenotype observations between Field Book and BreedBase without the need for manual file uploads. This walkthrough covers: Connecting Field Book to a BrAPI-enabled BreedBase server Importing trial layout and traits from BreedBase to Field Book Exporting collected phenotype data from Field Book to BreedBase Prerequisites: You must have a BreedBase user account with permissions to access trial and trait data. Your BreedBase instance must be configured with working BrAPI endpoints. The Field Book app must be installed on an Android device with internet access. Advantages of Using BrAPI: Reduces manual errors from file handling. Supports real-time syncing of data. Enables remote team coordination and efficient workflows. 12.7.1 Connecting Field Book to a BrAPI Server Walkthrough: Connecting Field Book to a BreedBase BrAPI Server 12.7.2 Importing Trials and Traits from BreedBase Walkthrough: Importing Trials from BreedBase via BrAPI Walkthrough: Importing Traits from BreedBase via BrAPI 12.7.3 Collecting Data Proceed with entering phenotype observations as described in the Collecting Phenotype Data in Field Book section. Ensure that: Trait values match expected formats (e.g., numeric, categorical). Observations are saved regularly during field use. 12.7.4 Exporting Phenotype Data to BreedBase via BrAPI Walkthrough: Sending Phenotype Data with BrAPI 12.7.5 Verifying Data in BreedBase After exporting data to BreedBase via BrAPI, it is important to verify that the export was successful directly in BreedBase. Log in to your BreedBase instance. Navigate to the Manage Trials or Trial Detail page for the uploaded study. Confirm that: Trait data appears in the Phenotype Data tab. Data points align with expected plot/plant identifiers. Trait units and scales match those defined in the database. 12.7.6 Troubleshooting Connection fails: Ensure you have the correct BrAPI server URL and valid credentials. Traits do not load: Confirm that the trait exists in BreedBase. Upload fails: Check for internet connectivity, correct user permissions, and trait formatting errors. "],["managing-phenotypic-data.html", "Chapter 13 Managing Phenotypic Data 13.1 Uploading Fieldbook Phenotypes 13.2 Uploading Spreadsheet Phenotypes", " Chapter 13 Managing Phenotypic Data To facilitate uploading process for phenotypic data, Manage Phenotypic Data page provides two options for uploading: Field Book Phenotype file in database format and phenotype file in Excel (.xls or .xlsx) file format. To access Manage Phenotypic Data page, clicking on Phenotyping in the Manage menu. 13.1 Uploading Fieldbook Phenotypes 13.1.1 Export Field Book Database File The database upload of Field Book phenotype data relies on the Database format from the Field Book. Please make sure to export the Database format from the Field Book if you intend to upload the data using the Field Book Upload we describe below. If you prefer to use the Table format that the Field Book exports, you can modify this format to work with the Speadsheet Upload we describe below. 13.1.2 Upload Field Book Database File To upload a Field Book Phenotype file in a database format, click the Upload Fieldbook link The Upload Fieldbook link on this page and Upload link on the Field Book Tools page open the same dialogue. Please follow instructions for uploading phenotypic files on the ?? page. 13.2 Uploading Spreadsheet Phenotypes To upload a phenotype file in an Excel (.xls or .xlsx) file format, click the Upload Spreadsheet link. Please specify Data Level (Plots or Plants) and select the Excel file that you want to upload. 13.2.1 Generating Spreadsheet File You can find more file format information by clicking on Spreadsheet Format link. Clicking on Spreadsheet Format will open the following dialog. Clicking on Create Phenotyping Spreadsheet will bring up a dialog where you can indicate the trial(s) you are interested in and the trait list you are interested in. Clicking Submit will download the xlsx file onto your computer, where you can then fill in the phenotypes. 13.2.2 Uploading Spreadsheet File To ensure that the file has a correct format for uploading, click on the Verify button. This will check the contents of the file and also perform quality checks on the values in the file. These checks include checking the trait definition for categorical values, minimum and maximum values, and data type checking. It will also check if there are already values uploaded for the given observation units and traits. If there are, there is an option to overwrite the existing values with the new values in your file. If the file is valid, only then can you click Store to store the information in the database. "],["managing-high-dimensional-phenotyping-data.html", "Chapter 14 Managing High Dimensional Phenotyping Data 14.1 Managing Transcriptomic Data", " Chapter 14 Managing High Dimensional Phenotyping Data 14.1 Managing Transcriptomic Data 14.1.1 Uploading Transcriptomic Data To upload transcriptomic data, go to the transcriptomics page by clicking Transcriptomics under the Manage tab. Click the Upload Transcriptomics Data button to open the upload workflow dialog. On the second step Samples, you will be prompted to make sure that your samples are already in the database. You can refer to the Managing Tissue Samples chapter for instructions on how to create a sampling trial. After creating a sampling trial or confirming that your samples exist, move on to the Protocol Info step. Here you can select an existing protocol, or create a new protocol. To create a protocol, click the Protocol Not Shown. Create a New Protocol button. From here you can fill in information about your new protocol and when finished, click Go to Next Step. You will need to have two csv files when uploading transcriptomics data. The first is the data matrix file, which should have these headers: The names in the sample_name column should match the sample names you created in your sampling trial file. The header marked transcript_name_columns should be replaced with the transcript names you want to upload, each in its own column in the header and the respective expression value below. Example: The second file is the transcript details file, which should have these headers: Each row under the transcript_name column should contain each transcript name from your data matrix file. For example: Once these two files are uploaded, click verify to check that they are formatted correctly and that the samples exist. If there are no errors, you can then click store to store your transcriptomics data. 14.1.2 Downloading Transcriptomics Data To download your data, click the Uploaded Transcriptomics Data dropdown menu on the Transcriptomics page. Then click the Your Uploaded Transcriptomic Data dropdown. From here you will be able to see the transcriptomics data that you have uploaded. Click the Download button next to the data that you wish to download. "],["managing-barcodes.html", "Chapter 15 Managing Barcodes", " Chapter 15 Managing Barcodes SGN databases provide tools for generating barcodes for stock identification. To access Barcode Tools page, clicking on Barcodes in the Manage menu. Barcode Tools page provides four options for generating barcodes: Single barcode Multiple barcodes Plot phenotyping barcodes Trial barcodes To generate single barcode, clicking on Generate Barcode link on the Barcode Tools page. In the Generate Barcode section, specify the name of the barcode, size of the barcode, then clicking on Generate Barcode The database will generate a barcode for your stock. The barcode can be printed for your stock identification. It also appears on its corresponding stock page. If you have a list of stocks that you want to generate barcodes, you can use Download Stock Barcodes section. You have three options for entering stock names: Typing in stock names, or copy and paste from other file into the box (1) Choosing a list of stocks from your Lists (2), and transferring the list into the box (1) by clicking on paste button. Uploading a Tab-delimited Text File with stock names. Select an optional printing format from the available formats. You can select printer settings that you prefer in the Printer Settings section. After you enter stock names and specify printer settings, clicking on Download Barcodes button at the bottom of the page. If you have a list of plots that you want to generate phenotyping barcodes, you can use Download Plot Phenotyping Barcodes section. You have three options for entering plot names: Typing in plot names, or copy and paste from other file into the box (1) Choosing a list of plots from your Lists (2), and transferring the list into the box (1) by clicking on paste button. Uploading a Tab-delimited Text File with plot names. If you have a list of trials that you want to generate barcodes, you can use Download Trial Barcodes section. You have three options for entering trial names: Typing in trial names, or copy and paste from other file into the box (1) Choosing a list of trial from your Lists (2), and transferring the list into the box (1) by clicking on paste button. Uploading a Tab-delimited Text File with trial names. "],["using-the-label-designer.html", "Chapter 16 Using the Label Designer 16.1 Select a Data Source 16.2 Set Page and Label Size 16.3 Design Your Label 16.4 Adjust Formatting, Save, and Download", " Chapter 16 Using the Label Designer Breedbase provides an interactive design tool for creating custom labels. To access the label design tool, click on Label Designer in the Manage menu. The following sections explain your many options as you advance through each step of the design workflow. 16.1 Select a Data Source The first step is to select a data source. To do so, you may directly select a specific data source from the Data Source dropdown menu (either a field trial, genotyping trial, or crossing trial) to populate your labels with the trial design information. Alternately, from the same dropdown menu, you may select a list to populate your labels with the list contents. Prior to making a selection from the Data Source dropdown menu, you may also filter the data sources by using the Datatype dropdown menu. You must then choose a level (plot, plant, etc.) before proceeding, using the Label for Every dropdown menu. To generate plot-level labels for more than one trial at once, select a list of trials as the source and plot as the level. Once you have made your selections, click the Next button to move to the next step in the workflow. 16.2 Set Page and Label Size After selecting the data source, you must choose whether to create a new design or load a saved design. If you choose the New selector, you will be prompted to select a page size and label size. If you do not see your page size or label size as an option, then select Custom from the relevant dropdown menu and enter your desired dimensions in pixels (each pixel is 1/72nd of an inch). If you choose the Saved selector, you will be prompted to select a saved design. After selecting a saved design, you will be taken directly to the design step with the saved design elements pre-loaded. 16.3 Design Your Label Having set the page and label formats, you will be presented with a label canvas where you can begin adding elements to your label. Select a type, field, size, and font, then click Add. You can add text to an existing field or create a completely custom field by clicking Create Custom Field. Once added, you can drag and drop elements, or you can delete them by clicking on the red box in their upper left corners. Barcodes can also be resized by dragging the green box in their lower right corners. If you are creating labels for a trial, it is highly recommended that you include a barcode encoding your plot, plant, or tissue sample names. These are the unique identifiers that will need to be included with any phenotypic or genotypic measurements loaded into the database. When you are satisfied with your label design, click Next. 16.4 Adjust Formatting, Save, and Download In the last step of the workflow, you can tweak your formatting and page layout, save your design, or download your labels. The Additional Settings dialog will allow you to adjust the page margins and margins between labels. The units are pixels (each pixel is 1/72nd of an inch). It is not recommended that you change these settings until you have already printed a test page. You can also set the number of copies per label, filter by replicate, or download only the first page for test purposes. To save your design, type a unique name in the text field and click Save. This will save your design to your list manager, where you can make it public in order to share it with others. Finally, if you are ready, click download to generate and download your labels! "],["managing-downloads.html", "Chapter 17 Managing Downloads", " Chapter 17 Managing Downloads You can download phenotype, trial meta-data, pedigree, GBS genotype and GBS genotype QC files from the database to your computer by using Lists. To download, clicking on Download in the Manage menu. For each category, you can select a list of accessions from your Lists to download their phenotypes, pedigree, GBS genotype, GBS genotype QC. In the case of downloading trial meta-data, you would provide a list of trials, while for downloading phenotype and GBS genotype QC, you can also use a list of trials or traits. "],["managing-odk-data-collection.html", "Chapter 18 Managing ODK Data Collection 18.1 ONA Crossing Information", " Chapter 18 Managing ODK Data Collection To access this page go to Manage and then ODK Data Collection. ODK is used for remotely collecting data on Android and IOS devices. We currently are working to support two ODK service providers, namely ONA and SMAP. We are using ONA to collect crossing information, including all lab activities following seed production. We are using SMAP for phenotypic data collection. 18.1 ONA Crossing Information 18.1.1 Managing ONA Crossing Information To begin collecting data using the ONA ODK form you must first have a crossing plan in the form of a Cross Wishlist. To do this from this page, click the Export Cross Wishlist to ONA button. Please refer to the Create Cross Wihlist help section for more information. It is possible to view the current available cross wishlists by clicking the Export Cross Wishlist to ONA button and then clicking Available Cross Wishlists. Once your cross wishlist is available, you can use your mobile ODK application to record crosses being done realtime. You can also record all laboratory activities following seed extraction up to greenhouse plantlet hardening. As you collect data using your mobile ODK application, your responses will be synchronized with our database. The Schedule Import for Selected Form section gives you options to perform the import daily or more frequently. It is also possible to initiate a data import from ONA at anytime by clicking Import Crossing Data from Selected Form on ONA. 18.1.2 Reviewing Plant Status The mobile ODK application has options to collect information about the status of plants in the field, such as if they are flowering. Images for each plant can also be recorded. The database will report this information here in a summary table that looks like the following. Notice that images are also transferred to the database. 18.1.3 Graphical Summary For Performed Crosses There is a section to summarize activities done for each cross. In this table each row represents a single cross performed. All the activities that have been performed will be shown here, such as first pollination and embryo rescue. The scatter plot shown tracks seed numbers generated on the Y axis and date of activity on the X axis. 18.1.4 Summary Information For Performed Crosses There is a secondary section to summarize what has been done across the entire Cross Wishlist. This tree structure shows all activities performed for a cross and shows how these crosses relate to the Cross Wishlist. "],["managing-tissue-samples.html", "Chapter 19 Managing Tissue Samples 19.1 Tissue samples from field trials 19.2 Genotyping Plate Tissue Samples (96 or 384 well plates)", " Chapter 19 Managing Tissue Samples To access this page go to Manage and then Tissue Samples. 19.1 Tissue samples from field trials A field trial contains plots planted with a specific accession. Each plot can contain many plants, which in turn can contain many tissue samples. On the manage tissue sample page we can see the field trials that contain tissue samples already. We can choose to download the tissue sample layout as seen in the below picture. If the field trial you want to collect tissue samples from is not in the above table, you can click the button highlighted below. Once you have clicked this button, you will enter a workflow that begins with the following introduction. Once you click next, you will need to select your trial. Next, if your trial currently only has plot entries saved, you will be asked to enter how many plants are in each plot. Finally you will be asked how many tissue samples you want for each plant. You can specify a string to include in the tissue sample name, such as leaf or root. Afterwards you should see the following success message, indicating that the tissue samples are saved. 19.2 Genotyping Plate Tissue Samples (96 or 384 well plates) A genotyping plate represents a 96 or 384 well plate. You can use the Coordinate Android application to create your plate layout, or you can upload your own Excel plate layout, or you can use the database to generate a plate layout. Ideally, you will use tissue sample names originating from a field trial as the source for each well tissue sample, but you can also use plant names, plot names, or accession names. From the manage tissue samples page, you can see the genotyping plates saved in the database. You can also download the layouts as shown below. If you need to create a new genotyping plate, you can click the button shown below. This will guide you through a workflow for uploading or creating the new plate layout. Genotyping vendors require you to send a plate layout during submission. You can download the plate layout as shown above, or you can go to a genotyping plate detail page to download the Intertek formatted file. In the future you will be able to directly export your genotyping plate plate layout to vendors. "],["managing-observation-variables.html", "Chapter 20 Managing Observation Variables 20.1 Managing Observation Variables with Traits, Methods, and Scales", " Chapter 20 Managing Observation Variables 20.1 Managing Observation Variables with Traits, Methods, and Scales Observation variables are the identifiers used when collecting phenotypic data. An observation variable is composed of a trait, a method, and a scale. The trait describes the attribute being measured e.g.Plant Height. The method defines the protocol in which the trait was observed e.g.Using a one meter long measuring stick. The scale defines the units or dimensions for which the measurement was taken e.g.Meters. Generally, observation variables are defined in ontologies that are predefined. We often use ontologies from cropontology.org. In this case, you will not be able to define your own observation variables directly; instead, you will need to contact us and we will add the observation variable for you. For databases where the user has greater control, we have an interface to allow addition of observation variables, along with traits, methods, and scales. To begin, go to the Search->Traits page. If the database you are on allows you to directly add observation variables, you will see the following button at the bottom of the page. When you click the button, the following workflow will appear. You should be logged in or else it will not allow addition of the observation variable. The workflow begins with an introduction. On the next workflow step, you select the ontology that you want to insert the new observation variable into. You must also give a name and a definition for the new observation variable. On the next workflow step, you select the trait ontology to use. Once you select a trait ontology, a select containing all the terms in the selected ontology will appear. You can either select a trait or if it does not exist in the select, you can create a new one by giving a name and a definition for the new trait. On the next workflow step, you select the method ontology to use. Once you select a method ontology, a select containing all the terms in the selected ontology will appear. You can either select a method or if it does not exist in the select, you can create a new one by giving a name and a definition for the new method. On the next workflow step, you select the scale ontology to use. Once you select a scale ontology, a select containing all the terms in the selected ontology will appear. You can either select a scale or if it does not exist in the select, you can create a new one by giving a name and a definition for the new scale. You can also define a format, minimum, maximum, categories, and default value for the new scale. On the last page of the workflow, you confirm the submission. Afterwards, you can use the newly created observation variable ontology term in your phenotyping. "],["managing-image-data.html", "Chapter 21 Managing Image Data 21.1 Image-Phenotyping Dashboard 21.2 Image Input 21.3 Standard Process 21.4 Ground Control Points", " Chapter 21 Managing Image Data 21.1 Image-Phenotyping Dashboard Upload raw image-captures in a compressed file (.zip) for orthophotomosaic assembly or upload previously stitched orthophotomosaic raster (.PNG, .JPG) imagery. Dashboard shows all field trials and uploaded imaging events in collapsible sections. Follow standard processes to manually create templates for assignment of plot-polygon images to the field experiment design. All imagery is shown with the spectral category within collapsible sections. Figure shows NIR imagery. Apply Fourier transform filtering, thresholding, and vegetation index masking. Plot-polygon images for all image processes are shown. Extract and export phenotypic values from plot-polygon images for analyses and model training. 21.2 Image Input Clicking Upload Imagery will open the following dialog. Raw-captures can be uploaded in a compressed (.zip) file so that they can be assembled into an orthophotomosaic. If orthophotomosaic assembly is not required, raster images (.PNG, .JPG) can be uploaded. Example data is given for raw Micasense RedEdge 5-band multispectral captures and for stitched orthophotomosaics. To begin uploading images, a field trial must be selected. The field trial must already be saved in the database. For information about adding a field trial, please read the Field Trial documentation. The image data is added to an imaging (drone run) event. Here you can select a previously saved imaging event or you can create a new one by defining a name, description, and date. The uploaded data can be raw image-captures or complete raster images. Here you can select whether orthophotomosaic stitching is required. In the case that orthophotomosaic stitching is required, select yes. On the next step you will see the following: Upload a zipfile with the raw-captures. When uploading Micasense RedEdge raw-captures, provide images of the Micasense calibration panels in a zipfile as well. In the case that orthophotomosaic assembly is not required, simple upload the raster images. Select the number of image bands that will be uploaded e.g.for a five band multispectral camera, select 5. In the caes that orthophotomosaic stitching is not required, select no. On the next step you will see the following: Upload an image at each band with a unique name, description, and spectral type. 21.3 Standard Process Once imagery is uploaded, it will appear on the dashboard under the field trial. Clicking the Run Standard Process button will begin extracting plot-polygon phenotypes from the imagery. Clicking the button will open the following dialog. Select a drone run band to use in this process. In the case of the Micasense 5 band multispectral camera there will be 5 bands shown here; select the NIR channel in this case because it has the highest contrast. In the case of standard color images, there will only be the RGB Color Image option here. Rotate the image so that there the plots are oriented in a grid fashion. There can be a skew in the field layout, as seen in the following example. Perform a rough cropping of the image by clicking on the four corners of the field. Cropping is important to remove any extraneous parts of the image. This step shows a histogram of the cropped image. The standard process will magnitude threshold the top and low ends of the distribution. In this step, the template for the plot polygons in the experimental field design are associated to the image. First, defined the number of rows and columns in the field experiment. Then click the four corners of the image, in respect to the top right, top left, botton left, and bottom right positions. Next click on Draw Plot Polygon Template. Review the template and clear/repeat the process until the template matches well. It is possible to copy/paste templates in the case where there are large breaks in the field design. Next, scroll down to the assign Plot Polygons to Field Trial Entities section. Select the location of Plot Number 1 as either top left or top right and whether the field design is serpentine or zigzag. Click on Generate Assignments and review that the names of the plots appear correctly in the overlay on the image. Finally, click Finish and Save Polygons to Plots when you have have confirmed the assignments. Next, the dialog shows you that the standard process will be repeated for all uploaded image bands. Next, choose which vegetation indices to apply. Next, choose the phenotypic values to extract. You must define the time point for which the phenotype is; if the field trial has a planting date, the time point will automatically be populated as image date minus the planting date. After completing the standard process, the job will continue in the background until it completes. You can check the status of the job from the dashboard. 21.4 Ground Control Points Ground control points can be saved after an imaging event has undergone the standard process on orhomosaics. Ground control points can then be used across imaging events on the same field experiment in order to automate the entire standard process. "],["managing-phenotypic-image-data.html", "Chapter 22 Managing Phenotypic Image Data 22.1 Uploading Image files 22.2 Uploading Images Directly to Stocks", " Chapter 22 Managing Phenotypic Image Data 22.1 Uploading Image files Clicking Phenotyping under Manage, and then Upload Images will open this dialog: If uploading images directly, select the file format Images. A single image or multiple selected images can be uploaded using this file format. Each of these images must have been downloaded from the Fieldbook app, or have a filename that follows this structure: observationUnitName, traitname, number, timestamp joined by underscores. The filename can not include any underscores other than the ones separating each value. For example: 38873_branching_5_2025-07-28.jpg If the images have been taken in the Fieldbook app, they will have EXIF data that includes their associated stocks and traits. This EXIF data will be automatically parsed and the associations will be made when the images are uploaded. After selecting your images for upload, clicking verify will check if your filenames or image metadata is correct and corresponds to existing stocks in the database. If the verification is successful, click store to save these images in the database. If uploading images with associated phenotypic data, select the file format Images with Associated Phenotypes. A spreadsheet with the phenotypic values needs to be uploaded alongside a zipfile of images. The spreadsheet can be in .xls or .xlsx format, and the column headers need to be: observationUnitName | observationVariableName | value | timestamp | image_name | person The image_name column should match the file names of the associated images in the zipfile. 22.2 Uploading Images Directly to Stocks In the Images section on a stock details page you can add a new image associated with that stock by clicking the Add Image button. This will associate any images uploaded through this dialog with the corresponding stock. "],["managing-vcf-data.html", "Chapter 23 Managing VCF Data 23.1 Uploading VCF Data 23.2 Searching and Downloading VCF Data 23.3 Searching Protocols 23.4 Detail Pages and Deletion", " Chapter 23 Managing VCF Data 23.1 Uploading VCF Data Genotyping data in VCF can be loaded from the web-interface. Breedbase can store any genotypic variants from a VCF, allowing for polyploids, structural variants, etc. without problems. To begin go to Manage->Genotyping Plates and click the button seen below: Note that you do not need to have genotyping plates uploaded to upload VCF data; you may upload genotyping data to accessions or you can upload genotyping data for tissue samples in genotyping plates. The workflow begins with an intro: On the following step in the workflow, a genotyping project is defined or selected. A genotyping project is a high-level entity for grouping several genotyping events. It is defined with a name, description, name, breeding program, and genotyping facility (IGD, Intertek, etc.). The following step is to define or select a genotyping protocol. A genotyping protocol represents the set of markers being called against a specific reference genome. A genotyping protocol is defined with a name, description, reference genome name, species name, and a location of data generation. Note in the picture that you can select whether the samples in your file are accessions or tissue samples in the database; tissue samples are for when a genotyping plate is stored in the database. There is an option to parse the sample names for appended sequencing numbers from IGD, where the sample names are like accession:igdnumber. The final step is to select the VCF from your computer and upload it. The web interface can be used to upload files arbitrarily large; it is a NGINX configuration to set this size. 23.2 Searching and Downloading VCF Data The Search Wizard is the primary means of querying data in the database. Go to Search->Wizard to begin. Once genotyping protocols are stored, select Genotyping Protocols from the first dropdown menu. Then if you select one or more and select Accessions from the second dropdown menu, you will see the accessions for which genotypes were stored. As seen in the following picture, there is a section for filtering genotypes by chromosome, start position, and end position. Genotypes can be downloaded in VCF or DosageMatrix formats. Using the Default genotyping protocol which is configured in a system, you can query over field phenotypic evaluations before downloading genotypes and phenotypes. 23.3 Searching Protocols Genotyping protocols can be search by going to Search->Genotyping Protocols. To download genotypes accessions must be selected, though any combination of search criteria can be used to filter and select those accessions. If a genotyping protocol is not selected, then the default genotyping protocol set in the configuration will be used. Genotyping protocols can also be selected in the wizard. The genotyping download menu on the Search Wizard presents options for filtering by chromosome, start position, and end position. Genotypes can be downloaded in VCF of Dosage Matrix formats. The genomic relationship matrix (GRM) can be downloaded for the selected accessions in a tab-delimited matrix format or in a three-column format that is useful in Asreml. Genotypes can be computed from the parents in the pedigree if those parents are genotyped by clicking on the compute from parents checkbox. Additionally, the GRM can be computed using genotypes of parents in the pedigree if the compute from parents checkbox is selected. As is described elsewhere, the Search Wizard presents a way to filter phenotypic values by minimum and maximum values, and allow for download in CSV and Excel formats. 23.4 Detail Pages and Deletion The genotyping protocol detail page will show all information about the protocol such as the reference genome used, the header information lines in the uploaded VCF file, the markers involved, and the samples genotyped. The markers section will show all markers used and their annotations, such as position, chromosome, alternate allele, reference allele, marker format, etc. The samples section will show all samples genotyped. Notice the Download links in the table which can be used to easily get the VCF file results for each genotyped samples with all markers in the genotyping protocol. For getting mulitple samples at once, use the Search Wizard as discussed above. The genotyping protocol and all associated genotyping data can be deleted from the genotyping protocol page. "],["managing-spectral-data.html", "Chapter 24 Managing Spectral Data 24.1 Upload Spectral Data 24.2 Evaluate and Remove Outliers 24.3 Plot Spectra 24.4 Aggregate Spectra 24.5 References", " Chapter 24 Managing Spectral Data Breedbase has implemented a flexible spectral data storage protocol that handles spectral data irrespective of the source spectrometer. Spectral data storage and analysis in Breedbase makes use of the R package waves for outlier identification, plotting, sample aggregation, and prediction model training. 24.1 Upload Spectral Data Spectral data can be added as a CSV file that includes metadata in the leftmost columns followed by one column per spectral measurement to the right. Rows represent a single scan or sample, each with a unique ID that must match to a Breedbase observationUnitName. Future data transfer using BrAPI will allow for interoperability with data collection software. To upload a spectral dataset, navigate to the Manage NIRS Data page by selecting NIRS in the Manage menu and click the blue Upload NIRS button. This will open an upload workflow. A link to the required file format and an example .csv file can be found by clicking in the light blue info box in this workflow. Another example of the file format is shown below. id: Optional identifier for each NIRS read. The id must be an integer. sampling_id: Optional identifier for each sample. Strings are allowed. sampling_date: Optional field. The format allowed is: YYYY-MM-DD. observationunit_name: Required field that matches existing data in the database. It can be the plot name, subplots, plant name, or tissue sample, depending how your trial is designed. device_id: Optional field to identify your device. Strings are allowed. device_type: Required field. It is possible upload data for a single device type. They can be: SCiO, QST, Foss6500, BunchiN500, or LinkSquare. comments: Optional field for general comments. All other columns are required wavelengths. You can add how many columns you want upload there is no limit. 24.2 Evaluate and Remove Outliers Spectral calibration models can be heavily affected by the presence of outliers, whether they come from spectrometer spectral artifacts or user errors. Mahalanobis distance (Mahalanobis, 1936) is a measure of the distance between a single observation and a larger distribution and is commonly used in the identification of outliers in a multivariate space (Des Maesschalck et al, 2000). The FilterSpectra() function in the R package waves calculates the Mahalanobis distance of each observation in a given spectral matrix using the stats::mahalanobis() function. Observations are identified as outliers if the squared distance is greater than the 95th percentile of a \\(\\chi\\)2-distribution with p degrees of freedom, where p is the number of columns (wavelengths) in the spectral matrix (Johnson and Wichern, 2007). In Breedbase, this procedure is applied on a per-dataset basis on upload and outliers are given binary tags Outlier. 24.3 Plot Spectra After outlier identification, a plot is generated using the PlotSpectra() function in waves. This function uses the filtered spectra and ggplot2::ggplot() to create a line plot with outliers highlighted by color. A list of rows identified as outliers are shown beneath the plot. Plots are saved as .png files and linked to the original input datasets. Plot image files can be downloaded via the Download Plot button in the upload workflow. 24.4 Aggregate Spectra To obtain a stable and reliable spectral profile, most spectrometer manufacturers recommend that multiple spectral scans are captured for each sample. While some spectrometers aggregate these scans internally, many do not, requiring the user to do so before analysis can take place. Breedbase handles these cases upon data upload following filtering steps by calling the AggregateSpectra() function from waves, saving the aggregated scans for future access through the search wizard feature. Scans are aggregated by sample mean (e.g.plot-level basis) according to the provided observationUnitName field. After aggregation, the user exits the upload workflow and the raw data file is saved in the upload archive. 24.5 References De Maesschalck, R., Jouan-Rimbaud, D., and Massart, D. L. (2000). The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. Johnson, R. A. & Wichern, D. W. (2007). Applied Multivariate Statistical Analysis (6th Edition). p 773. Mahalanobis, P. C. (1936). On the generalized distance in statistics. National Institute of Science of India. Analysis tool documentation "],["managing-sequence-metadata.html", "Chapter 25 Managing Sequence Metadata 25.1 What is Sequence Metadata? 25.2 Loading Sequence Metadata 25.3 Searching Sequence Metadata 25.4 Marker Integration 25.5 Sequence Metadata API", " Chapter 25 Managing Sequence Metadata 25.1 What is Sequence Metadata? Sequence Metadata is a feature that allows for the efficient storage and retrieval of sequence annotations for a specific region along a reference genome. The annotation data can contain a primary score value and any number of secondary key/value attribute data. For example, Sequence Metatadata can store MNase open chromatin scores for every 10 basepairs along the reference genome as well as genome-wide association study (GWAS) statistics, including the trait information associated with the result. This data can then be filtered by position and/or scores/attribute values and even cross-referenced with markers stored in the database. 25.2 Loading Sequence Metadata Sequence Metadata can be loaded into the database using a gff3-formatted file. The following columns are used to load the data: #1 / seqid: The name of the database feature (ie chromosome) the metadata is associated with (The feature name must already exist as a feature in the database) #4 / start: The metadatas start position #5 / end: The metadatas end position #6 / score: (optional) The primary score attribute of the metadata #9 / attributes: (optional) Secondary key//value attributes to be saved with the score. These should be formatted using the gff3 standard (key1=value1;key2=value2). The attribute key cannot be either score, start, or end. To upload the gff3 file: Go to the Manage > Sequence Metadata page Click the Upload Sequence Metadata button On Step 2 of the Wizard, select the Type of data to be uploaded This groups similar datasets together in the same Data Type category On Step 3 of the Wizard, select an existing Protocol or create a new one The Protocol is used to describe how the data was generated and define the score value and any secondary attributes. Adding the attributes (and their descriptions) to the Protocol will allow the Sequence Metadata queries to filter the data based on the value of one or more of these attributes. Attributes not defined in the Protocol will still be stored and displayed on retrieval, but will not be able to be used in a search filter. Finally, select and upload your gff3 file to the database. The database will verify the format of the file before its contents are stored. 25.3 Searching Sequence Metadata To retrieve stored Sequence Metadata, go to the Search > Sequence Metadata page. 25.3.1 Basic Search The basic Sequence Metadata search options include selecting the reference genome and species, the chromosome, and (optionally) the start and/or end position(s) along the reference genome. In addition, one or more specific protocols can be selected to limit the results. The Sequence Metadata search results are returned as a table, including the chromosome and start/stop positions of the annotation, along with the primary score value and any additional key/value attributes. The markers column will include a list of marker names of any stored markers that are found within the start/stop positions of the Sequence Metadata. The data can be downloaded as a table in an Excel or CSV file or a machine-readable (code-friendly) JSON file. If the Sequence Metadata JBrowse configuration is set, the filtered results can be displayed as a dynamic JBrowse track. 25.3.2 Advanced Search Any number of advanced search filters can be applied to the query. The advanced filters can limit the search results by the value of the primary score and/or any of the secondary attribute values. 25.4 Marker Integration A table of Sequence Metadata annotations are embedded on the Marker/Variant detail page. The table will include any annotations that span the poisiton of the marker (for data of the same reference genome and species). 25.5 Sequence Metadata API A publicly accessible RESTful API (Application Programming Interface) is available to query the database for Sequence Metadata directly from your programming environment (R, python, etc) to be used in analysis. The data is returned in a JSON format. Documentation for the API can be found on the Manage > Sequence Metadata page "],["managing-outliers-in-dataset.html", "Chapter 26 Managing Outliers in Dataset 26.1 What is Outliers Functionality in Dataset ? 26.2 Accessing Trait Visualization 26.3 Interpreting Visual Elements 26.4 Choosing Cut-Off Values 26.5 Setting Deviation Multiplier 26.6 Utilizing Graph Controls", " Chapter 26 Managing Outliers in Dataset 26.1 What is Outliers Functionality in Dataset ? As in step The Search Wizard we can create a dataset. The dataset incorporates a feature to identify outlier points, which we may choose to exclude from a specific dataset. Its important to note that these exclusions only apply at the dataset level, and no data is permanently removed from the database. Additionally, outlier categorization can be modified at any time, and these changes are visible to all other functionalities within the system. Each dataset stores a wholly unique set of outlier points, completely independent of any other dataset in the database. Outliers are specifically designated for traits within datasets, exclusively encompassing phenotype data. If a particular dataset lacks traits as a part of wizard selection, this functionality is not available. Each trait has its own set of defined outliers. 26.2 Accessing Trait Visualization Once youve selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. 26.3 Interpreting Visual Elements Once youve selected a specific trait, the web application provides access to a visualization of the data points associated with that trait. Green Points: As per the legend, represent values for the selected trait that fall below the cut-off point set by the slider. (non-outliers) Black Outlined Points: These data points are outlined with black borders, indicating that they are currently designated as outliers in the database. Red Points: The red data points denote the cut-off points established by the slider for the allowable deviation value. 26.4 Choosing Cut-Off Values You have two fundamental options for setting cut-off points: Median with MAD: This option involves using the median (middle value) along with the Mean Absolute Deviation (MAD) as a reference point for determining cut-off values. Mean with Standard Deviation: Alternatively, you can choose to use the mean (average) in conjunction with the Standard Deviation to set cut-off points. 26.5 Setting Deviation Multiplier The slider allows you to specify the deviation multiplier from a central point, which influences the cut-off values. 26.6 Utilizing Graph Controls Beneath the graph, youll find four buttons, each serving a distinct function: Add selection to outliers: This button enables you to save the current cut-off points to the database for future reference. Reset outliers for current trait: You can use this option to reset outliers for the selected trait. Reset all outliers: This button allows you to reset outliers for the entire dataset. Download Phenotype Table without outliers: You can download the phenotype data table in a comma-separated value format file, using this feature, with outliers excluded for selected dataset. These tools and functions are designed to provide you with control and insights when working with data visualization and outliers. "],["data-analysis-tools.html", "Chapter 27 Data Analysis Tools 27.1 Selection Index 27.2 Genomic Selection 27.3 Genome Browsing 27.4 Principal Component Analysis (PCA) 27.5 ANOVA 27.6 Clustering (K-Means, Hierarchical) 27.7 Genetic Gain 27.8 Kinship and Inbreeding Coefficients 27.9 Creating Crossing Groups 27.10 Search Wizard Genomic Relationship Matrix (GRM) Download 27.11 Search Wizard Genome Wide Association Study (GWAS) 27.12 Spectral Analysis 27.13 General Mixed Model Tool 27.14 Genomic Prediction of Cross Performance (GPCP) 27.15 Tool Compatibility", " Chapter 27 Data Analysis Tools SGN databases provides several tools for phenotype data analysis, marker-assisted selection, sequence and expression analyses, as well as ontology browser. These tools can be found in the Analyze menu. 27.1 Selection Index To determine rankings of accessions based on more than one desirable trait, SGN databases provide a Selection Index tool that allows you to specify a weighting on each trait. To access the tool, clicking on Selection Index in the Analyze menu. On the Selection Index page, selecting a trial that you want to analyze. After you selected a trial, you can find traits that were assayed in that trial in the Trait box. Selecting a trait that you want to include in the analysis will open a new dialogue showing the selected trait and a box that you can assign a Weight of that trait. After you are done, you can continue by selecting another trait by clicking on Add another trait link. After you selected another trait, this page will automatically update information for you by showing all of the traits that you selected for the analysis. You also have options to choose a reference accession, choose to include accessions with missing phenotypes, scaling values to a reference accession. After you complete your setting, clicking on Calculate Rankings The Selection Index tool will generate rankings of accessions based on the information that you specified. You can copy the results to your system clipboard, convert the table data to CSV format, or print the data. Clicking on Raw Average will display average values of the phenotypes of those ranked accessions. Selection Index tool also allows you to save top ranked accessions directly to Lists. You can retrieve top ranked accessions by selecting a number or a percent. 27.2 Genomic Selection The prediction of breeding values for a trait is a one step or two steps process, depending on what stage in your breeding cycle you are. The first step is to build a prediction model for a trait using a training population of clones with phenotype and genotype data. If you have yet to select parents for crossing for your first cycle of selection you can use the breeding values of the training population. If you are at later stages of your selection program, you need to do the second step which is applying the prediction model on your selection population. All clones in your training and selection populations must exist in the database. To use the genomic selection tool, on cassavabase.org, select Genomic Selection from the analyze pull-down menu. There are three ways to build a model for a trait. 27.2.1 Building a Model - Method 1: One way to build a model is, using a trait name, to search for trials in which the trait was phenotyped and use a trial or a combination of trials to build a model for the trait. For example, if you search for mosaic disease severity, you will get a list of trials you can use as training populations. You will get a list of trials (as shown below) in which the trait of your interested was phenotyped. From the list, you can use a single trial as a training population or combine several trails to form a training population for the prediction model of the trait. Lets say, you want to create a training population using individuals from trials cassava ibadan 2001/02 and cassava ibadan 02/03 and build a model for cassava mosaic disease severity using all clones from the training population. Select the trials to combine (the same coloured), click done selecting, click the combine trials and build model button, and you will get a model and its output for the trait. On the model detail page, you can view the description of input data used in the model, output from the model and search interface for selection populations the model you can apply to predict their breeding values. The description of the input data for the model includes the number of phenotyped clones, and the number of markers, scatter and frequency distribution plots for the phenotype data, relationship between the phenotype data and GEBVs, population structure. The model output includes model parameters, heritability of the trait , prediction accuracy, GEBVs of the individuals from the training population and marker effects. Expand each section to see detailed information. If you expand the Trait phenotype data section, you will find plots to explore the phenotype data used in the model. You can assess the phenotype data using a scatter and histogram plots and the descriptive statistics. A regression line between observed phenotypes and GEBVs shows the relationship between the two. You can also explore if there is any sub-clustering in the training population using PCA. To check the model accuracy, a 10-fold cross-validation test, expand the model accuracy section. Marker effects are also available for download. To do so, expanad the Marker Effects section and click the Download all marker effects link and you will get a tab delimited output to save on your computer. The breeding values of the individuals used in the training population are displayed graphically. Mousing over each data point displays the clone and its breeding value. To examine better, you can zoom in into the plot by selecting an area on the plot. You can download them also by following the Download all GEBVs link. Estimating breeding values in a selection population If you already have a selection population (in the database), from the same model page, you can apply the model to the selection population and estimate breeding values for all the clones in the population. You can search for a selection population of clones in the database using the search interface or you can make a custom list of clones using the list interface. If you click the search for all relevant selection populations, you will see all relevant selection populations for that model. However, this option takes long time decause of the large set of populations in the database and the filtering. Therefore, the fastest way is to search for each of your selection populations by name. If you are logged in to the website you will also see a list of your custom set of genotyped clones. To apply the model to a selection population, simply click your population name or Predict Now and you will get the predicted breeding values. When you see a name of (or acronym]) of the trait, follow the link and you will see an interactive plot of the breeding values and a link to download the breeding values of your selection population. 27.2.2 Building a Model - Method 2 Another way to build a model is by selecting a trial, instead of selecting and searching for a specific trait. This approach is useful when you know a particular trial that is relevant to the environment you are targeting to breed material for. This method allows you to build models and predict genomic estimated breeding values (GEBVs) for several traits within a single trial at once. You can also calculate selection index for your clones when GEBVs are estimated for multiple traits. To do this select the Genomic Selection link found under the analyze menu. This will take you to the same home page as used with Method 1. However, instead of entering information to search for in Search for a trait, click on Use a trait as a trial population. This will expand a new menu that will show all available trials. To begin creating the model, select the existing trial that you would like to use. In this example I will be using the trial and trait data from Cassava Ibadan 2002/03 trial. Clicking on a trial will take you to a page where you can find information such as number of markers and number of phenotypes clones. In addition to the number of phenotype clones and number of markers, the main page for the trial selected also has information and graphs on phenotypic correlation for all of the traits. By moving your cursor over the graph you can read the different values for correlation between two traits. A key with all of the trait names of the acronyms used can be found in the tab below the graph. Below the Training population summary there is a tab for Traits. Clicking on this tab will show all available traits for the specific trial. You can create a model by choosing one or multiple traits in the trial and clicking Build Model. In this example, the traits for cassava bacterial blight severity and cassava mosaic disease severity have been selected. Clicking on Build Model will take you to a new page with the models outputs for the traits. Under the Genomic Selection Model Output tab you can view the model output and the model accuracy. Clicking on any of the traits will take you to a page with information about the model output on that individual trait within the trial. There you can view all of the trait information that was seen in more detail in Method 1. You can apply the models to simultaneously predict GEBVs for respective traits in a selection population by clicking on Predict Now or the name of the selection population. You can also apply the models to any set of genotyped clones that you can create using the lists feature. For more information on lists, click here. Follow the link to the trait name to view and download the predicted GEBVs for the trait in a selection population. To compare clones based on their performance on multiple traits, you can calculate selection indices using the form below. Choose from the pulldown menu the population with predicted GEBVs for the traits and assign relative weights for each trait. The relative weight of each trait must be between 0 - 1. 0 being of least weight and importance, not wanting to consider that particular trait in selecting a genotype and 1 being a trait that you give highest importance. In this example we will be using the Cassava Ibadan 2002/03 population and assigning values to each of the traits. Remember that there is a list of acronyms and trait names at the bottom of the page for reference. After entering whatever values you would like for each trait click on the Calculate button to generate results. This will create a list of the top 10 genotypes that most closely match the criteria that you entered. The list will be displayed right below the selection index tab. This information can also be downloaded onto your computer by clicking on the Download selection indices link underneath the listed genotypes and selection indices. 27.2.3 Building a Model - Method 3 In addition to creating a model by searching for pre-existing traits or by preexisting trial name, models can also be created by using your own list of clones. This creates a model by using or creating a training population. The page to use the third Method for creating a population model is the same as for the other two models. Select Genomic Selection from under the analyze menu of the main toolbar. This will take you to the Genomic Selection homepage and show you all three available methods to create a model. To see and use Method 3 scroll down and click on the tab labeled Create a Training Population. This will open a set of tools that will allow you to use pre-existing lists or to create a new list. Once the Create a Training Population tab is opened you have the option to use a pre-existing list or create new one. To learn how to create a list, click here. The Make a new list of plots link will take you directly to the Search Wizard that is usually used to create lists. Please note: the only lists that can be used in Method 3 to create a model are lists of plots and trials. If the pre-existing list is not of plots or trials (for example, traits, or locations) it will not show up and cannot be used as a training population. When you create you use a list of trials, the trials data will be combined to create a training data set. To use your custom list of plots or trials as a training population, select the list and click Go. This will take you to a detail page for the training population. From here on you can build models and predict breeding values as described in Method 2. 27.3 Genome Browsing There are two ways to evaluate genotype information within the browser, from an accession detail page or a trial detail page. 27.3.1 Browsing Genotype data by Accession If you are interested in browsing genotype information for a single accession, for example BAHKYEHEMAA, navigate to the accession detail page. Near the bottom of the detail page is a collapsible section called Accession Jbrowse. This section will contain a link to the accession jbrowse page if the necessary genotype data is available. Clicking the link should take you to a page that looks like this, a which point you can browsre the genotype data in the form of a vcf track aligned to the latest build of the genome. 27.3.2 Browsing Genotype data by Trial If you are interested in browsing genotype information for the accessions within a given trial, navigate to the trial detail page. Halfway down the page is a collapsible section called Trial Jbrowse. This section will contain a link to the trial jbrowse page if the necessary genotype data for at least two accessions planted in the trial is available. Clicking the link should take you to a page that looks like this, a which point you can browse the genotype data in the form of vcf tracks aligned to the latest build of the genome. 27.4 Principal Component Analysis (PCA) Principal component analysis helps estimate and visualize if there is sub-grouping of individuals within a dataset based on a number of variables. Currently, you can use marker data to run PCA on datasets. You can run PCA from multiple places on the website. To do PCA on individuals from a trial, go to the trial detail page and find the PCA tool under the Analysis tools section. individuals from a training population you used in a GS modeling, do your modeling and find the PCA tool in the model output page. individuals in a training population and selection population you applied the training model, do your modeling, apply the model on the selection population and find the PCA tool on the selection population prediction output page. individuals in a list of accessions you created, for example using the search wizard, go to the Analyze menu and select the Population Structure, select your list of individuals and run PCA. individuals from multiple trials, create a list of the trials using the search wizard, go to the Analyze menu and select the Population Structure, select your list of trials and run PCA. With all the options, you will get a interactive plot of the two PCs (shown below) that explain the largest variance. Point the cursor at any data point and you will see the individual name with its corresponding PCs scores. By clicking the Download all PCs, you can also download the 10 PCs scores in the text format. 27.5 ANOVA Currently, ANOVA is implemented for a single trial (single year and single location). You can do ANOVA for RCBD, CRD, Alpha and Augmented trial designs. ANOVA is done using linear mixed effects model, where the genotypes is fixed effect and the replications and blocks are random effects. Fixed effect significance level is computed using lmer from lmeTest R package. You can do ANOVA from two places: trial detail and training population detail. In both cases, if the phenotype data was from the supported trial designs, Go to the ANOVA section down in the trial or training population page Select the trait of you want to perform ANOVA Click the Run ANOVA and wait for the result 27.6 Clustering (K-Means, Hierarchical) The K-Means method allows you to partition a dataset into groups (K number). The hierarchical clustering, agglomerative, allows you to explore underlying similarity and visualize in a tree structure (dendrogram) the different levels of similarities (clusters) among samples. You can do clustering based on marker data, phenotype data and GEBVs. When you use phenotype data, first clone averages for each trait are calculated. Both methods use Euclidean distance as a measure of similarity. For the hierachical clustering, the complete-linkage (farthest neighbour) method is used to link up clusters. There are three pathways to using this tool. When you have data in the form of a list or dataset from the search wizard: go to the Analyze menu and select the clustering option make sure you are logged in Select the relevant genotyping protocol, if you are clustering using genotype data select your list or dataset, click Go select clustering type select the data type to use If you are running K-Means clustering, provide the number of partitions (K). If left blank it will partition the data set into optimal numbers for the dataset. click the Run Cluster and wait for the analysis to finish or queue the request and wait for an email with the analysis result. You can download the outputs following the download links. From the trial detail page: Go to the Analysis Tools section Follow steps D to G in (1) In the solGS pipeline: Once you you are in a model output put page, you will see a section where you can do clustering in the same way as above (option 2). K-Means clustering: Hierarchical clustering: 27.7 Genetic Gain You can check for genetic gain by comparing the the GEBVs of a training and a selection population. You can do this in the solGS pipepline once you build a model and apply the model to predict the GEBVs of a selection population. Once at that stage, you will see a section Check Genetic Gain. Select a selection population to compare with the training population and click the Check Genetic Gain button. The genetic gain will be visualized in boxplots. You can download the boxplot(s) as well as the GEBVs data used for the plot(s). 27.8 Kinship and Inbreeding Coefficients This tool allows you to estimate genetic relatedness between a pair of individuals (kinship), homozygousity across loci in an individual (inbreeding coefficient), and genetic similarity of an individual relative to the rest of the population (averge kinship). There are three pathways to using this tool. (1) When you have a list or dataset clones, created from the search wizard: go to the Analyze menu and select the kinship and inbreeding make sure you are logged in Select the genotypic protocol for the marker data select your list or dataset of clones, click Go click the Run Kinship and wait for the analysis to finish, depending on the data size this may take minutes. You can choose to submit the analysis and wait for an email notice to view the results or wait for it to complete. You can download the output following the download links. (2) From the trial detail page: Go to the Analysis Tools section Follow steps C to G in (1) (3) In the solGS pipeline: Once you you are in a model output put page, scroll down to the Kinship and Inbreeding section and run kinship. 27.9 Creating Crossing Groups If you calculate selection index based on GEBVs of multiple traits, and you want to select a certain proportion of the indexed individuals (e.g.top 10%, or bottom 10%) and then you want to partition the selected individuals into a number of groups based on their genotypes, you can use the k-means clustering method. The procedure is: predict GEBVs for 2 or more traits In the models output page, calculate selection indices. Note the name of the selection index data. Go to the clustering section, select the selection index data, select K-means, select Genotype, in the K-numbers textbox, fill in the number of groups you want to create, in the selection proportion textbox, fill in the proportion of the indexed individuals you want to select, e.g.for the top 15 percent, 15. if you wish to select bottom performing, prefix the number with minus sign (e.g.-15) then run cluster and wait for the result. 27.10 Search Wizard Genomic Relationship Matrix (GRM) Download The genomic relationship matrix (GRM) is useful for understanding underlying structure in your population. Breedbase can compute the GRM using rrBLUP. First, select accessions in the search wizard and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. The GRM can be returned in a matrix format (.tsv) which shows all pairwise relationships between the selected accessions and is useful for visualization; alternatively, the GRM can be returned in a three-column format (.tsv) which is useful for programs like ASReml outside of Breedbase. The GRM can also be returned as a simple correlation heatmap image (.pdf). The GRM can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox compute from parents; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. 27.11 Search Wizard Genome Wide Association Study (GWAS) Performing a genome wide association study (GWAS) can determine genotypic markers which are significantly correlated to phenotypic traits. Breedbase can compute GWAS using rrBLUP. First, select accessions and trait(s) in the search wizard, and optionally select a genotyping protocol. If no genotyping protocol is selected, the default genotyping protocol in your system is used (as defined in sgn_local.conf). Several traits can be selected in the search wizard; if the traits are not to be treated as repeated measurements then select no in the select box and this will tell Breedbase to return GWAS results independently for the selected traits. If the selected traits are indeed all repeated measurements then select yes in the select box and Breedbase will return as single GWAS analysis across all the phenotypic records. Specify the minor allele frequency, missing marker data, and missing individuals data filters to apply. GWAS results can be returned in a tabular format (.tsv) where the -log10(p-values) for the selected traits are returned; alternatively, the GWAS results can be returned as Manhattan and QQ plots for the selected traits. The GWAS can be computed from parents of the selected accessions granted the parents were genotyped, by clicking the checkbox compute from parents; this is useful for programs where parental lines are genotyped and then hybrids are created and evaluated in the field. The GWAS will filter the data by the input MAF and missing data filters provided. After filtering the data is imputed using an EM method in rrBLUP. The Kinship matrix (GRM) is computed from the imputed genotypic data and used in the GWAS model. The GWAS uses fixed effects for different field trials and replicates in the phenotypic data. 27.12 Spectral Analysis Visible and near-infrared spectroscopy (vis-NIRS) can be related to reference phenotypes through statistical models to produce accurate phenotypic predictions for unobserved samples, increasing phenotyping throughput. This technique is commonly used for predicting traits such as total starch, protein, carotenoid, and water content in many plant breeding programs. Breedbase implements the R package waves to offer training, evaluation, storage, and use of vis-NIRS prediction models for a wide range of spectrometers and phenotypes. 27.12.1 Dataset selection In order to initiate an analysis, the user must select one or more datasets using 2.1. A dataset in Breedbase can contain observationUnit-level (plot-, plant-, or sample-level) trial metadata and phenotypic data from one or more trials. After navigating to the NIRS webpage under the Manage tab in Breedbase, the user can initiate an analysis and select one of these datasets as input for model training. An optional test dataset can be selected in the second step of the workflow. 27.12.2 Cross-validation Five cross-validation schemes that represent scenarios common in plant breeding are available for this analysis. These include CV1, CV2, CV0, and CV00 as outlined below and described in depth by Jarqun et al.(2017) as well as random and stratified random sampling with a 70% training and 30% validation split. For those schemes from Jarqun et al.(2017), specific input datasets must be chosen based on genotype and environment relatedness. Cross-validation choices: * Random sampling (70% training / 30% validation) * Stratified random sampling, stratified based on phenotype (70% training / 30% validation) * CV1, untested lines in tested environments * CV2, tested lines in tested environments * CV0, tested lines in untested environments * CV00, untested lines in untested environments 27.12.3 Preprocessing Preprocessing, also known as pretreatment, is often used to increase the signal to noise ratio in vis-NIR datasets. The waves function DoPreprocessing() applies functions from the stats and prospectr packages for common spectral preprocessing methods with the following options: * Raw data (default) * First derivative * Second derivative * Gap segment derivative * Standard normal variate (SNV; Barnes et al., 1989) * Savitzky-Golay polynomial smoothing (Savitzky and Golay, 1964) For more information on preprocessing methods and implementation, see the waves manual, available through CRAN: waves.pdf 27.12.4 Algorithms Several algorithms are available for calibration model development in Breedbase via the waves package. The TrainSpectralModel() function in waves performs hyperparameter tuning as applicable using these algorithms in combination with cross validation and train functions from the package caret. Currently, only regression algorithms are available, but classification algorithms such as PLS-DA and SVM clasification are under development. * Partial least squares regression (PLSR; Wold et al., 1982; Wold et al., 1984) is a popular method for spectral calibrations, as it can handle datasets with high levels of collinearity, reducing the dimensionality of these data into orthogonal latent variables (components) that are then related to the response variable through a linear model (reviewed in Wold et al., 2001). To avoid overfitting, the number of these components included in the final model must be tuned for each use case. The PLSR algorithm from the pls package is implemented by waves. * Random Forest regression (RF; Ho, 1995) is a machine learning algorithm based on a series of decision trees. The number of trees and decisions at each junction are hyperparameters that must be tuned for each model. Another feature of this algorithm is the ability to extract variable importance measures from a fitted model (Breiman, 2001). In Breedbase, this option is made available through implementation of the RF algorithm from the package randomForest in the waves function TrainSpectralModel(). This function outputs both model performance statistics and a downloadable table of importance values for each wavelength. It is worth noting that this algorithm is computationally intensive, so the user should not be alarmed if results do not come right away. Breedbase will continue to work in the background and will display results when the analysis is finished. * Support vector machine regression (SVM; Vapnik, 2000) is another useful algorithm for working with high-dimension datasets consisting of non-linear data, with applications in both classification and regression. The package waves implements SVM with both linear and radial basis function kernels using the kernlab package. 27.12.5 Output: common model summary statistics After training, model performance statistics are both displayed on a results webpage and made available for download in .csv format. These statistics are calculated by the TrainSpectralModel() function in waves using the caret and spectacles packages. Reported statistics include: * Tuned parameters depending on the model algoritm * Best.n.comp, the best number of components to be included in a PLSR model * Best.ntree, the best number of trees in an RF model * Best.mtry, the best number of variables to include at every decision point in an RF model * RMSECV, the root mean squared error of cross-validation * R2cv, the coefficient of multiple determination of cross-validation for PLSR models * RMSEP, the root mean squared error of prediction * R2p, the squared Pearsons correlation between predicted and observed test set values * RPD, the ratio of standard deviation of observed test set values to RMSEP * RPIQ, the ratio of performance to interquartile distance * CCC, the concordance correlation coefficient * Bias, the average difference between the predicted and observed values * SEP, the standard error of prediction * R2sp, the squared Spearmans rank correlation between predicted and observed test set values 27.12.6 Export model for later use Once a model has been trained, it can be stored for later use. This action calls the SaveModel() function from waves. Metadata regarding the training dataset and other parameters specified by the user upon training initialization are stored alongside the model object itself in the database. 27.12.7 Predict phenotypes from an exported model (routine use) For phenotype predictions, users select a dataset and can then choose from models in the database that were trained using the same spectrometer type as the spectral data in the chosen dataset. Predicted phenotypes are stored as such in the database and are tagged with an ontology term specifying that they are predicted and not directly measured. Metadata regarding the model used for prediction is stored alongside the predicted value in the database. Predicted phenotypes can then be used as normal in other Breedbase analysis tools such as the Selection Index and GWAS. 27.12.8 FAQ The Breedbase Spectral Analysis Tool does not allow for prediction models involving data from multiple spectrometer types at once. References * Barnes, R.J., M.S. Dhanoa, and S.J. Lister. 1989. Standard normal variate transformation and de-trending of near-infrared diffuse reflectance spectra. Appl. Spectrosc. 43(5): 772-777. doi: 10.1366/0003702894202201. * Breiman, L. 2001. Random forests. Mach. Learn. 45: 5-32. doi: 10.1201/9780429469275-8. * Ho, T.K. 1995. Random decision forests. Proc. Int. Conf. Doc. Anal. Recognition, ICDAR 1: 278-282. doi: 10.1109/ICDAR.1995.598994. * Jarqun, D., C. Lemes da Silva, R.C. Gaynor, J. Poland, A. Fritz, et al.2017. Increasing Genomic-Enabled Prediction Accuracy by Modeling Genotype x Environment Interactions in Kansas Wheat. Plant Genome 10(2): plantgenome2016.12.0130. doi: 10.3835/plantgenome2016.12.0130. * Johnson, R.A., and D.W. Wichern. 2007. Applied Multivariate Statistical Analysis (6th Edition). De Maesschalck, R., D. Jouan-Rimbaud, and D.L. Massart. 2000. The Mahalanobis distance. Chemom. Intell. Lab. Syst. 50(1): 1-18. doi: 10.1016/S0169-7439(99)00047-7. * Mahalanobis, P.C. 1936. On the generalized distance in statistics. Natl. Inst. Sci. India. * Savitzky, A., and M.J.E. Golay. 1964. Smoothing and Differentiation of Data by Simplified Least Squares Procedures. Anal. Chem. 36(8): 1627-1639. doi: 10.1021/ac60214a047. * Shrestha, R., L. Matteis, M. Skofic, A. Portugal, G. McLaren, et al.2012. Bridging the phenotypic and genetic data useful for integrated breeding through a data annotation using the Crop Ontology developed by the crop communities of practice. Front. Physiol. 3 AUG(August): 1-10. doi: 10.3389/fphys.2012.00326. * Vapnik, V.N. 2000. The Nature of Statistical Learning Theory. Springer New York, New York, NY. * Wold, S., A. Ruhe, H. Wold, and W.J. Dunn, III. 1984. The Collinearity Problem in Linear Regression. The Partial Least Squares (PLS) Approach to Generalized Inverses. SIAM J. Sci. Stat. Comput. 5(3): 735-743. doi: 10.1137/0905052. * Wold, S., M. Sjstrm, and L. Eriksson. 2001. PLS-regression: a basic tool of chemometrics. Chemom. Intell. Lab. Syst. 58(2): 109-130. doi: 10.1016/S0169-7439(01)00155-1. 27.13 General Mixed Model Tool The general mixed model tool is available at /tools/mixedmodels and a link is provided from the Analyze menu. To use the mixed model tool, first create dataset using the Wizard containing the data that you would like to analyze. Select the Mixed Model tool from the Analyze menu. You are presented with a workflow. On the first step of the workflow, select the dataset that you wish to analyze, click on Choose dataset to continue. The second part of the workflow presents you with the traits in the dataset; you can select one or more traits from the lists using the select buttons. If you selected one trait, a bargraph of the trait distribution will be shown. Click the Next step button to move to the next screen. On the model build screen, all the factors are displayed that are contained within the dataset. The factors are presented as a list of blue buttons that can be dragged using the mouse to areas on the screen which build a mixed model equation. The areas correspond to fixed factors, random factors, and optionally to more complex factors, such as fixed factors with interaction and fixe factors with vriable slope/intersects. Drag the available factors to the corresponding area. To calculate BLUPs for germplasm, drag the germplasmName button to the Random factors area. To calculate BLUEs, drag it to the Fixed factors area. The factors need to have different levels contained within them, for example, if there is only one trial in the dataset, it cannot be used as one of the factors. Click on Run analysis and got to next step to run the mixed model and display the results. The result view contains two tabs, one with the raw data, either BLUPS or BLUEs, and the other the adjusted means from the raw data. The results can be stored in the database as an analysis, by clicking the button provided on the top of the data. 27.14 Genomic Prediction of Cross Performance (GPCP) The GPCP tool is available at /tools/gcpc and a link is provided from the Analyze menu. The GCPC tool implements genomic prediction with additive and directional dominance in the linear mixed model to predict for cross performance. Before using the tool, first create a dataset using the Wizard containing the data that you would like to analyze. (The dataset should have genotyping_protocols). Second, create Selection Indices for your traits using Selection Index in Analyze Menu. To use the tool, Select the GPCP tool from the Analyze menu. Then, select the dataset with genotyping_protocols that you wish to analyze, click on Proceed to Factor Selection to load available factors that can be included in the model. Select the factors you wish to include in the model either as Fixed or Random. Click None for factors that you dont want to include in the model. Note that the germplasmName is factored as Random by default. The next step is to select the selection index for your traits on the dropdown menu. Once you are through, click Run GPCP to run the model. The output will be presented in form of a table with ID, Parent1, Parent2 and their cross prediction merit organized in descending order. The results will also have sex information based on whether the dataset has plant sexes available in the database. 27.15 Tool Compatibility The dataset definition enables one to predict whether the dataset can be used in various analysis tools. Upon creating a dataset, the site will automatically predict its compatibility with the available analysis tools and report these values on the dataset details page. In the table, each tool will report to the user which traits are available to be analyzed based on phenotype data, and if different types of analyses are available, these will also be reported to the user. Some tools may give a warning sign to indicate that this dataset is compatible, but with potentially low sample sizes. Hover over the warning symbol to get a readout of the reason for the warning. Below the table, there is a button that enables the user to re-calculate tool compatibility. This can be useful if a dataset is created before phenotypes are uploaded to a trial, since phenotype data is used in determining dataset compatibility. Even if the page appears to hang, do not worry; the compatibility check will continue in the background, and you can check later. Below the tool compatibilities, there is also a summary of the data encompassed by the dataset and the criteria used for determining tool compatibility. Those criteria are used in the following way: - Correlation: A dataset can be used in a correlation analysis if there are many phenotype measurements for different traits made on the same accession. - Population Structure (PCA): A genotype PCA can be run if there are many accessions all genotyped with the same protocol. A phenotype PCA can be run if many accessions all have measurements on many traits. - Clustering: Like a PCA, clustering can be done in both phenotype and genotype modes. They have the same requirements as PCA. - Kinship & Inbreeding: A dataset with many accessions genotyped with the same protocol can be used for kinship analyses. - Stability: A dataset containing many accessions with the same trait measured across multiple locations can be used in stability analyses. - Heritability: This requires one or more trials with the same trait measured on the same accession across those trial(s). - Mixed Models: This requires sufficient accession numbers, trait measurements, and trial designs. - GWAS: A dataset is compatible with GWAS if there are many accessions genotyped for the same genotyping protocol, and the genotyping protocol has enough markers to run a GWAS. In addition, each accession needs to be phenotyped for a trait. - Boxplotter: There must be sufficient trait measurements to make a boxplot of the trait. In addition to being on the dataset details page, tool compatibilities may be listed on the dataset selection screens for analysis tools. The compatibilities are non-blocking; you may always try using a dataset in an analysis even if there are warnings or if it is deemed non-compatible. As before, you can hover over the warning symbols to see why a dataset may not have statistical power. For analyses with multiple modes, such as clustering and PCA, you can also hover over the compatibliity checkmark to see what types (phenotype or genotype) the dataset is compatible with. "],["vector-viewer.html", "Chapter 28 Vector Viewer", " Chapter 28 Vector Viewer BreedBase provides a vector viewer tool that can generate, save, or upload vectors onto BreedBase. The vector viewer tool comes with many interactive features: Highlighting Location display Zooming in / panning out A feature and restriction enzyme table that support real time editing, deleting, or adding To highlight a region of the vector viewer, click and drag your cursor clockwise. To delete the new highlight, either start a new highlighted segment by clicking on the vector, or click the Update/Reset button. To find the location of a spot on the vector, hover your cursor over the vector and a number will appear. That is the location, in base pairs, of your cursor. To zoom in and pan out, click the + or the - button underneath the vector. To add a feature or restriction site, click Add. To edit a feature or restriction site, select the feature or restriction site of choice, click Edit, and make any wanted changes. To delete a feature or restriction site, selected the feature or restriction site that you want to delete and then click Delete. "],["404.html", "Page not found", " Page not found The page you requested cannot be found (perhaps it was moved or renamed). You may want to try searching to find the page's new location, or use the table of contents to find the page you are looking for. "]] diff --git a/docs/searching-the-database.html b/docs/searching-the-database.html new file mode 100644 index 0000000000..2397a134c7 --- /dev/null +++ b/docs/searching-the-database.html @@ -0,0 +1,574 @@ + + + + + + + Chapter 2 Searching the Database | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +
    + + +
    +
    + +
    +
    +

    Chapter 2 Searching the Database

    +

    You can search for information on the database by using the following search options: Wizard, which uses combined criteria specified by users; Accessions and Plots; Trials; Markers; Images; People; FAQ.

    +

    +
    +

    2.1 The Search Wizard

    +

    +
    +

    2.1.1 How the Search Wizard Works

    +

    The search wizard presents a number of select boxes, which are initially empty. You start searching by picking a category of data from the dropdown above the left-most select box.

    +

    Once a category has been picked, the database will retrieve all the options within this category and display them within the first select box. You then select one or more options from the first select box, which activates the second dropdown.

    +

    You can then select a category from the second dropdown, and repeat this same search process through all four dropdowns and select boxes.

    +

    +
      +

    • In the example above, the “locations” category was chosen in the first dropdown. The first select box then displayed all the possible locations in the database. The option Ibadan was selected.

      • +

    • This activated the second dropdown. The category “years” was chosen in the second dropdown. The second select box then displayed all the years that are linked in the database to the location Ibadan. From that list, the options 2011 and 2012 were selected.

      • +

    • This activated the third dropdown. A final category, “accessions”, was chosen in the third dropdown. The third select box was then populated with the 3847 accessions in the database that are linked with the location Ibadan in the years 2011 or 2012.

      • +
    +

    In addition to the basic search operations demonstrated above, users can take advantage of two more features:

    +

    Load Selection from List

    +

    +
      +
    • Instead of picking a category in the first dropdown, users can instead populate the first selectbox from a list by scrolling down in the first dropdown to the “Load Selection from List” subheading and selecting a list. This is useful for starting queries with a list of plots, as this category is not among the options in the first dropdown.
      • +
    +

    ANY/MIN/ALL Toggle

    +

    +
      +

    • By default, the search wizard combines options within a category using an OR query. In the example above, in the third panel the wizard retrieved accessions associated with the location ‘Ibadan’ in ANY of the years “2011 OR 2012”

      • +

    • If the user clicked the toggle below the second select box to change it to ALL before choosing accessions in the third dropdown, the wizard would instead retrieve accessions associated with the location ‘Ibadan’ in the years “2011 AND 2012”. This will be a smaller set of accessions, because any accessions used only in 2011, or only in 2012 will be excluded.

      • +

    • A more advanced search could use the MIN toggle option. This allows the user to make a query in between an ANY or ALL query, where a minimum number of matches from the selected column will be used as a filter for the next column. The minimum can be provided as either a percentage (%) or an actual count of items (#). In the example above, if the years 2011, 2012, and 2013 were selected in the second column, the user could enter ‘2’ in as the minimum and select ‘#’ as the minimum match type. This would select accessions in the third column that were used in 2 or more of the selected years.

      • +
    +

    +
    +
    +

    2.1.2 How to use retrieved data

    +
    +

    Getting more Info

    +

    Any option in the wizard select boxes (except for years) can be clicked to open a page with more details. The new page is opened in a new tab.

    +
    +
    +

    Saving to a list

    +

    You can store the highlighted items in any selected box to lists. This is done using the inputs and buttons directly below the select box. Don’t forget, you must be logged in to work with lists!

    +

    +
      +

    • To add items to an existing list, first pick an existing list using the “Add to List…” dropdown on the left. Then click the “Add” button. A popup window will confirm the action, and display the number of items added to your existing list.

      • +

    • To store items to a new list, first type a new list name in the “Create New List…” text input on the left. Then click on the “Create” button. A popup window will confirm the action, and display the number of items added to your new list.

      • +
    +
    +
    +

    Downloading Data

    +

    You can download trial metadata, phenotypes and genotypes associated with the highlighted items in the wizard select boxes. This is done using the buttons in the download section at the bottom of the page. Don’t forget, you must be logged in to download data!

    +

    +
    +
    Metadata
    +

    Trial metadata can be downloaded by selecting a subset of trials from the database or based on your search categories. To download, click on “Related Trial Metadata”, a dialog will appear. Select download format and click the “Metadata” button to complete your download.

    +

    +
    +
    +
    Phenotypes
    +

    The phenotypes download is quite flexible, and can download a subset of all the trial data in the database based on whichever categories and options you currently have selected. Simply click on the “Related Trial Phenotypes” link, review the options, changing or adding any additional parameters you like, then click ‘Download Phenotypes’.

    +

    +
    +
    +
    Genotypes
    +

    The genotype download is more stringent. It requires a minimum of one accession and one genotyping protocol to be selected in the wizard select boxes. The text box in the download section of the page will help track what has been selected. Once clicked, the “Download Genotypes” button will download a genotype file for the selected accessions.

    +
    +
    +
    +

    Saving the wizard selections

    +

    As discussed above, the selections of the individual select boxes in the wizard can be saved separately to a list. The lists can be used as inputs in other tools on the site. However, sometimes creating a selection is quite time consuming and restoring the selections from four different lists would be cumbersome too. Therefore, the selections can be saved together in a dataset, and named for later retrieval. This is done in the section “Load/Create Datasets” that is below the first two wizard select boxes. To select an existing dataset, one uses the “Load Dataset” dropdown. A particular dataset can be chosen, and the “Load” button can be clicked to retrieve and display the dataset in the wizard. To create a new dataset using items that are selected in the wizard, one can enter the name of the new dataset in the “Create New Dataset” text box. Once the dataset has been given a name, clicking the “Create” button will save the new dataset.

    +

    +
    +
    +
    +

    2.1.3 Updating the Wizard

    +

    The search wizard uses a copy of the database, or a cache, to return results quickly. If data appears to be missing, it usually means that the cache needs to be updated. Users with submitter privileges or above can do this using the ‘Update Wizard’ button. One can also use the ‘Refresh Lists’ button to update the available lists.

    +

    +

    This will take just a few seconds in small databases, but may take a few hours to complete in larger databases.

    +
    +
    + + + +
    +

    2.5 Ontology Browser

    +

    A more advanced tool for searching for Traits is the ontology browser, available by clicking on Analyze and Ontology Browser. From here you can search ontologies and see the various classifications of terms in a tree display.

    +

    +

    The terms which appear in the Trait Search in 2.4 are only variable terms. The ontology browser shows these variables as different from their grouping terms by indicating VARIABLE_OF like in the following screenshot.

    +

    +
    +
    +

    2.6 Search Seedlots

    +

    Seedlots are different from Accessions in that they represent the physical seed being evaluated in an experiment. Seedlots have things like physical storage locations and seed quantities, which accessions do not. To search for available seedlots you go to Manage and then click Seed Lots. By clicking Search Seedlots, you can specify query information. The results from your search will be in the table below the search form.

    +

    +

    + +
    +
    +
    + +
    +
    +
    + + +
    +
    + + + + + + + + + + + + + + + diff --git a/docs/using-fieldbook-app.html b/docs/using-fieldbook-app.html new file mode 100644 index 0000000000..a0fb5cf394 --- /dev/null +++ b/docs/using-fieldbook-app.html @@ -0,0 +1,651 @@ + + + + + + + + Chapter 12 Using Field Book and Breedbase for Data Collection and Storage | User Manual of Breedbase + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +
    + + +
    +
    + +
    +
    +

    Chapter 12 Using Field Book and Breed Base for Data Collection and Storage

    +

    +
    +

    12.1 Introduction

    +

    Field Book is an open-source Android app designed to replace traditional paper field books for collecting phenotypic data. By switching to digital data collection with Field Book, users eliminate the need for manual data transcription and reduce the risk of errors. Field Book requires two files for collecting data: a Field Layout file and a trait file.

    +

    Breed Base supports phenotypic data collection and export using the Field Book app for trials stored in its database. This support includes tools for generating field layout and trait files for manual upload, as well as automated data transfer through BrAPI.

    +
    +
    +

    12.2 Downloading and Installing Field Book

    +
    +

    12.2.1 Downloading from Google Play

    +

    The app is available from the Google Play Store here or scan the following QR code on your Android device:

    +

    +

    For detailed instructions on installing from the Play Store see:

    + +
    +
    +

    12.2.2 Installing Manually from GitHub

    +

    Add Scribe.

    +
    +
    +

    12.2.3 Updating the Application

    +

    It is recommended to keep Field Book updated to ensure your app has latest bug fixes and features available. For detailed instructions, see:

    + +
    +
    +
    +

    12.3 Creating and Manually Uploading Field Layout and Trait Files to Field Book

    +
    +

    12.3.1 Creating Field Layout Files in BreedBase

    +
    +
    +

    12.3.2 Importing Field Layout Files to Field Book

    +
    +
    +

    12.3.3 Creating Trait Files in BreedBase

    +
    +
    +

    12.3.4 Importing Trait Files to Field Book

    +
    +
    +
    +

    12.4 Collecting and Exporting Phenotype Data

    +
    +

    12.4.1 Collecting Phenotype Data in Field Book

    +
    +
    +

    12.4.2 Exporting Phenotype Data from Field Book to BreedBase

    +
    +
    +
    +

    12.5 Using BrAPI for Data Transfer Between Field Book and BreedBase

    +
    +
    +

    12.6 A typical workflow

    +
      +

    1. Creating a field layout file based on the design of field trial

      2. +

    2. Creating a trait file from the list of traits

      3. +

    3. Downloading the field layout file and trait file from the database to your computer

      4. +

    4. Downloading the field layout file and trait file to the tablet (where the Field Book App is installed)

      5. +

    5. Collecting phenotypes

      6. +

    6. Exporting phenotypes from Field Book App to your computer

      7. +

    7. Uploading the exported phenotype file from your computer to the database

      8. +
    +
    +
    +

    12.7 Creating Field Layout Files for the Field Book App

    +

    There are two alternative methods for creating “Field Layout Files”.

    +
      +

    1. Using “Field Book Tools” page

      2. +

    2. Using “Trial Detail” page.

      3. +
    +
    +

    12.7.1 Creating “Field Layout Files” by using “Field Book Tools” page.

    +

    To access “Field Book Tools” page, clicking on “Field Book App” in the “Manage” menu.

    +

    +

    On the “Field Book Tools” page, clicking on “New”

    +

    +

    On the “Download Fieldbook” window, selecting trial name and data level (plots or plants), then clicking on “Submit” button. A treatment can be selected, which allows you to record phenotypes based on treatment application. A list of traits can be selected, which provides a summary of an accession’s global performance for those traits in the Fieldbook.

    +

    +

    If the field book layout file was successfully created, a pop-up window will indicate that the field book layout file was saved successfully. Clicking on the file name will immediately download the file onto your computer. The file is also available to download on the “Field Book Tools” page, if you need to re-download it.

    +

    +

    To download field layout file to your computer, clicking on “Download File”, the file can then be transferred to your tablet. If you no longer want to keep the field layout file, clicking on “Delete Layout File”.

    +

    +
    +
    +

    12.7.2 Creating “Field Layout Files” by using “Trial Detail” page.

    +

    To create “Field Layout Files”, go to the “Trial Detail” page of the trial that you want to create the file. On the “Trial Detail” page, scrolling down to the bottom of the page to find “Android Field Book Layout” in the “Files” section, then clicking on the “Create Field Book” link.

    +

    +

    Clicking on the “Create Field Book” link will open a new window showing the name of the trial that you selected, as well as data level (plots or plants). A treatment can be selected, which allows you to record phenotypes based on treatment application. A list of traits can be selected, which provides a summary of an accession’s global performance for those traits in the Fieldbook. To proceed, clicking on “Submit” button.

    +

    +

    If the field book layout file was successfully created, a pop-up window will indicate that the field book layout file was saved successfully. Clicking on the file name will immediately download the file onto your computer. The file is also available to download on the “Field Book Tools” page, if you need to re-download it.

    +

    +

    To download field layout file to your computer, clicking on “Download File”, the file can then be transferred to your tablet. If you no longer want to keep the field layout file, clicking on “Delete Layout File”.

    +

    +
    +
    +
    +

    12.8 Creating Trait Files for the Field Book App

    +

    Steps to Create a Trait File:

    +
    +

    12.8.1 Creating a Trait List

    +

    After you logged in, lists can be created and managed using the Search Wizard or the “Lists” link. For more information on how to create lists, click here.

    +

    +
    +
    +

    12.8.2 Creating a Trait File

    +

    After you have your trait list, clicking on the “Field Book App” link found under the “Manage” menu tab. This will take you to the “Field Book Tools” page.

    +

    +

    To create a new trait file, finding the heading “Trait Files”, then clicking on the “New” link.

    +

    +

    Clicking on the “New” link will open a dialogue box titled “Create Trait File”. Please enter your “Trait file name” and select “List of traits to include” from drop-down list that you previously created. You can only use traits included in the list. Check the box titled “Include Notes Trait” if you would also like to record and upload general plot notes in the field. Click “OK” to submit.

    +

    +

    If your trait file was successfully created, a new window will indicate that the trait file was saved, then clicking on “Close”.

    +

    +

    After the trait file was saved, you will see your file listed in the “Field Book Tools” page. Clicking on “Download” link to download the trait file to your computer.

    +

    +

    After downloading the trait file to your computer, the file can be transferred to an Android Tablet. You need the Android Field Book App to open the file. The Android Field Book App can be downloaded at: http://www.wheatgenetics.org/bioinformatics/22-android-field-book

    +
    +
    +
    +

    12.9 Transferring Files from Your Computer to Android Tablet

    +
    +

    12.9.1 Files on your computer

    +

    After downloading, Field Layout files and Trait files can be found in the “Downloads” folder of your computer. Field Layout files on your computer will have a prefix “fieldbook_layout_” added to the beginning of the file name. For example: “2014-01-28_19:14:34_Trial Demo_location 6767.xls” on the the database website will be saved as “field_book_layout_2014-01-28_19:14:34_Trial Demo_location 6767.xls” on your computer.

    +

    +

    The files can be transferred to Android tablet by copying the files into the tablet’s Internal Storage File.

    +
    +
    +

    12.9.2 Files on your Android tablet

    +

    To transfer Field Layout file and Trait file to your Android tablet, connecting an Android tablet to your computer, then clicking on tablet icon on your computer. Clicking on the tablet icon will open a window showing an “Internal Storage” file.

    +

    +

    After you installed the Android Field Book App, all files for the app are stored in the “fieldBook” folder within the “Internal storage” folder.

    +

    +

    Within the “fieldBook” folder, there are five sub-folders:

    +
      +

    • field_export

      • +

    • field_import

      • +

    • plot_data

      • +

    • resources

      • +

    • trait

      • +
    +

    Field Layout files must be copied into the “field_import” folder.

    +

    +

    Trait files must be copied into the “trait” folder.

    +

    +

    You can either drag and drop, or copy the Field Layout file and the Trait file from your computer to the folders in your Android tablet.

    +
    +
    +
    +

    12.10 Setting up “Field Book App” for data collection

    +

    After you transferred the Field Layout file and Trait file from your computer to Android tablet, you still need to set up “Field Book App” on your tablet for data collection.

    +

    To set up the Field Book App:

    +
      +
    1. To open the Field Book App in the Android Tablet, clicking on the Field Book App icon, which is a green rectangle.
      2. +
    +

    +
      +
    1. To import Field Layout files, clicking on the “Fields” section of the main menu of the Field Book App.
      2. +
    +

    +

    Clicking on the “Fields” tab will open a new dialogue that will let you select the file that you want to import.

    +

    +

    Choosing a Field File will generate a new dialogue that will ask you to choose between an Excel or CSV format. Since the data from the database is in Excel format, choose the Excel option.

    +

    +

    After submitting the file format, a final dialogue box will appear. Please provide information about the file that you want to import. Please ensure that “plot_name” is set as the unique identifier. To finalize the process, clicking “OK” button.

    +

    +
      +
    1. To import Trait Files, clicking on the “Traits” tab on the main menu of the Field Book App.
      2. +
    +

    +

    Then, clicking on the three dots symbol found on the upper right corner of the Field Book screen. This will open a drop down menu with the choices “Import” and “Export”. Clicking on “Import”

    +

    +

    Clicking on “import” will open a new dialogue that displays a list of trait files that you can select to import to the Field Book App.

    +

    +

    The trait file is now imported into the Field Book App. The traits page will show all trait files and available traits.

    +

    +
    +
    +

    12.11 Exporting Files from Field Book App

    +

    Data that were collected on the Field Book App can be exported back to your tablet folder, which can then be transferred to your computer.

    +

    To export files containing data from the Field Book App to your tablet, clicking on the “Export” link on the main menu page of the Field Book App.

    +

    +

    Clicking on the “Export” link will open a new dialogue window. To ensure that data are exported in a correct format for the database, checking the “Database Format” box, then clicking on “OK” button.

    +

    +

    The exported file can then be found in the “field_export” sub-folder within the “fieldBook” folder on your tablet. Once you connect your tablet to your computer, you can directly transfer the file to your computer.

    +

    +

    +
    +
    +

    12.12 Uploading Phenotype Files to an SGN database

    +

    To upload phenotype files to the database, clicking on “Field Book App” in the “Manage” menu.

    +

    +

    On the “Field Book Tools” page, clicking on “Upload” link in the “Uploaded Phenotype Files” section.

    +

    +

    Clicking on the “Upload” link will open a new dialogue asking you to choose a file that you want to upload to the database website. Please ensure that “plot_name” is the first column of the file to be uploaded. To make sure that the file has the correct format for uploading, click on the “Verify” button. After the file format has been verified, click on the “Store” button.

    +

    +

    The list of uploaded phenotype files can be found on the Field Book Tools page

    +

    +

    The uploaded files will also be seen in the corresponding “Trial Detail” page.

    +

    + +
    +
    +
    + +
    +
    +
    + + +
    +
    + + + + + + + + + + + + + diff --git a/docs/using-the-label-designer.html b/docs/using-the-label-designer.html new file mode 100644 index 0000000000..59b053b3d4 --- /dev/null +++ b/docs/using-the-label-designer.html @@ -0,0 +1,491 @@ + + + + + + + Chapter 16 Using the Label Designer | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +
    + + +
    +
    + +
    +
    +

    Chapter 16 Using the Label Designer

    +

    Breedbase provides an interactive design tool for creating custom labels. To access the label design tool, click on “Label Designer” in the “Manage” menu. The following sections explain your many options as you advance through each step of the design workflow.

    +
    +

    16.1 Select a Data Source

    +

    The first step is to select a data source. To do so, you may directly select a specific data source from the “Data Source” dropdown menu (either a field trial, genotyping trial, or crossing trial) to populate your labels with the trial design information. Alternately, from the same dropdown menu, you may select a list to populate your labels with the list contents. Prior to making a selection from the “Data Source” dropdown menu, you may also filter the data sources by using the “Datatype” dropdown menu. +You must then choose a level (plot, plant, etc.) before proceeding, using the “Label for Every” dropdown menu. To generate plot-level labels for more than one trial at once, select a list of trials as the source and plot as the level. Once you have made your selections, click the ‘Next’ button to move to the next step in the workflow.

    +

    +
    +
    +

    16.2 Set Page and Label Size

    +

    After selecting the data source, you must choose whether to create a new design or load a saved design. If you choose the “New” selector, you will be prompted to select a page size and label size. +If you do not see your page size or label size as an option, then select “Custom” from the relevant dropdown menu and enter your desired dimensions in pixels (each pixel is 1/72nd of an inch). +If you choose the “Saved” selector, you will be prompted to select a saved design. After selecting a saved design, you will be taken directly to the design step with the saved design elements pre-loaded.

    +

    +
    +
    +

    16.3 Design Your Label

    +

    Having set the page and label formats, you will be presented with a label canvas where you can begin adding elements to your label. Select a type, field, size, and font, then click “Add”. +You can add text to an existing field or create a completely custom field by clicking “Create Custom Field”. +Once added, you can drag and drop elements, or you can delete them by clicking on the red box in their upper left corners. +Barcodes can also be resized by dragging the green box in their lower right corners. +If you are creating labels for a trial, it is highly recommended that you include a barcode encoding your plot, plant, or tissue sample names. +These are the unique identifiers that will need to be included with any phenotypic or genotypic measurements loaded into the database. +When you are satisfied with your label design, click “Next”.

    +

    +
    +
    +

    16.4 Adjust Formatting, Save, and Download

    +

    In the last step of the workflow, you can tweak your formatting and page layout, save your design, or download your labels. +The “Additional Settings” dialog will allow you to adjust the page margins and margins between labels. The units are pixels (each pixel is 1/72nd of an inch). It is not recommended that you change these settings until you have already printed a test page. +You can also set the number of copies per label, filter by replicate, or download only the first page for test purposes. +To save your design, type a unique name in the text field and click “Save”. This will save your design to your list manager, where you can make it public in order to share it with others. +Finally, if you are ready, click download to generate and download your labels! +

    + +
    +
    +
    + +
    +
    +
    + + +
    +
    + + + + + + + + + + + + + + + diff --git a/docs/vector-viewer.html b/docs/vector-viewer.html new file mode 100644 index 0000000000..2f45807411 --- /dev/null +++ b/docs/vector-viewer.html @@ -0,0 +1,479 @@ + + + + + + + Chapter 28 Vector Viewer | Breedbase User Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +
    + + +
    +
    + +
    +
    +

    Chapter 28 Vector Viewer

    +

    BreedBase provides a vector viewer tool that can generate, save, or upload vectors onto BreedBase.

    +

    +

    The vector viewer tool comes with many interactive features:

    +
      +

    • Highlighting

      • +

    • Location display

      • +

    • Zooming in / panning out

      • +

    • A feature and restriction enzyme table that support real time editing, deleting, or adding

      • +
    + + +

    To highlight a region of the vector viewer, click and drag your cursor clockwise. To delete the new highlight, either start a new highlighted segment by clicking on the vector, or click the “Update/Reset” button.

    +

    +

    To find the location of a spot on the vector, hover your cursor over the vector and a number will appear. That is the location, in base pairs, of your cursor.

    +

    +

    To zoom in and pan out, click the “+” or the “-” button underneath the vector.

    +

    +

    +

    To add a feature or restriction site, click “Add”. To edit a feature or restriction site, select the feature or restriction site of choice, click “Edit”, and make any wanted changes. To delete a feature or restriction site, selected the feature or restriction site that you want to delete and then click “Delete”.

    +

    + + + +
    +
    + +
    +
    +
    + + +
    +
    + + + + + + + + + + + + + + + diff --git a/forms/primer3/formfu_create.yaml b/forms/primer3/formfu_create.yaml index 2ab7aa6b81..aef5070ec1 100644 --- a/forms/primer3/formfu_create.yaml +++ b/forms/primer3/formfu_create.yaml @@ -2,7 +2,7 @@ # indicator is the field that is used to test for form submission indicator: submit # Start listing the form elements -action: update +action: primer3/update elements: - type: Block @@ -21,13 +21,21 @@ elements: label: "Paste source sequence below (5'->3', string of ACGTNacgtn -- other letters treated as N -- numbers and blanks ignored). FASTA format ok. Please N-out undesirable sequence (vector, ALUs, LINEs, etc.) or use a Mispriming Library (repeat library):" name: PRIMER_MISPRIMING_LIBRARY attributes: - title: "This selection indicates what mispriming library (if any) Primer3 should use to screen for interspersed repeats or for other sequence to avoid as a location for primers. The human and rodent libraries on the web page are adapted from Repbase (J. Jurka, A.F.A. Smit, C. Pethiyagoda, et al., 1995-1996) ftp://ftp.ncbi.nih.gov/repository/repbase). The human library is humrep.ref concatenated with simple.ref, translated to FASTA format. There are two rodent libraries. One is rodrep.ref translated to FASTA format, and the other is rodrep.ref concatenated with simple.ref, translated to FASTA format. + title: > "This selection indicates what mispriming library (if any) Primer3 should use to screen for interspersed repeats or for + other sequence to avoid as a location for primers. The human and rodent libraries on the web page are adapted from Repbase + (J. Jurka, A.F.A. Smit, C. Pethiyagoda, et al., 1995-1996) ftp://ftp.ncbi.nih.gov/repository/repbase). The human library + is humrep.ref concatenated with simple.ref, translated to FASTA format. There are two rodent libraries. One is rodrep.ref + translated to FASTA format, and the other is rodrep.ref concatenated with simple.ref, translated to FASTA format. -The Drosophila library is the concatenation of two libraries from the Berkeley Drosophila Genome Project: + The Drosophila library is the concatenation of two libraries from the Berkeley Drosophila Genome Project: - 1. A library of transposable elements The transposable elements of the Drosophila melanogaster euchromatin - a genomics perspective J.S. Kaminker, C.M. Bergman, B. Kronmiller, J. Carlson, R. Svirskas, S. Patel, E. Frise, D.A. Wheeler, S.E. Lewis, G.M. Rubin, M. Ashburner and S.E. Celniker Genome Biology (2002) 3(12):research0084.1-0084.20, http://www.fruitfly.org/p_disrupt/datasets/ASHBURNER/D_mel_transposon_sequence_set.fasta + 1. A library of transposable elements The transposable elements of the Drosophila melanogaster euchromatin - a genomics + perspective J.S. Kaminker, C.M. Bergman, B. Kronmiller, J. Carlson, R. Svirskas, S. Patel, E. Frise, D.A. Wheeler, + S.E. Lewis, G.M. Rubin, M. Ashburner and S.E. Celniker Genome Biology (2002) 3(12):research0084.1-0084.20, + http://www.fruitfly.org/p_disrupt/datasets/ASHBURNER/D_mel_transposon_sequence_set.fasta - 2. A library of repetitive DNA sequences http://www.fruitfly.org/sequence/sequence_db/na_re.dros. " + 2. A library of repetitive DNA sequences http://www.fruitfly.org/sequence/sequence_db/na_re.dros. " + options: - [ '', 'NONE' ] - [ 'static/documents/primer3-libraries/human.txt', 'HUMAN' ] diff --git a/js/build.webpack.config.js b/js/build.webpack.config.js index 6de08bbe81..58382ea2ec 100644 --- a/js/build.webpack.config.js +++ b/js/build.webpack.config.js @@ -1,7 +1,7 @@ const path = require('path'); const glob = require("glob"); const filemap = require(path.resolve(__dirname,"./webpack_util/webpack-filemap-plugin.js")); -const UglifyWebpackPlugin = require("uglifyjs-webpack-plugin"); +const TerserPlugin = require("terser-webpack-plugin"); const sourcePath = path.resolve(__dirname, "source"); const entryPath = path.resolve(sourcePath, "entries"); @@ -39,7 +39,8 @@ module.exports = { options: { sourceType: "unambiguous", presets: [['@babel/preset-env',{ - useBuiltIns: 'usage' + "useBuiltIns": "usage", + "corejs": "3" }]] } },{ @@ -59,12 +60,7 @@ module.exports = { optimization: { minimize: true, namedChunks: true, - minimizer: [new UglifyWebpackPlugin({ - uglifyOptions: { - output: { - ascii_only: true - }, - }, + minimizer: [new TerserPlugin({ 'sourceMap': true, 'parallel': 4, })], diff --git a/js/install_node.sh b/js/install_node.sh index 4a259f77af..3d0e963c56 100644 --- a/js/install_node.sh +++ b/js/install_node.sh @@ -9,7 +9,7 @@ fi curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - # Add the desired NodeSource repository -VERSION=node_10.x +VERSION=node_20.x # The below command will set this correctly, but if lsb_release isn't available, you can set it manually: # - For Debian distributions: jessie, sid, etc... # - For Ubuntu distributions: xenial, bionic, etc... diff --git a/js/package.json b/js/package.json index 955c4b9bbb..68a7bce6ac 100644 --- a/js/package.json +++ b/js/package.json @@ -14,33 +14,33 @@ "author": "", "license": "MIT", "devDependencies": { - "@babel/core": "^7.3.3", - "@babel/polyfill": "^7.2.5", - "@babel/preset-env": "^7.3.1", - "babel-loader": "^8.0.5", + "@babel/core": "^7", + "@babel/preset-env": "^7", + "babel-loader": "^8", "deepmerge": "^2.2.1", "del": "^3.0.0", - "jsdom": "^13.2.0", + "jsdom": "^16", "loader-utils": "^1.2.3", "nock": "^10.0.6", "node-fetch": "^2.3.0", - "requirejs": "2.3.6", + "requirejs": "2.3.7", "save-svg-as-png": "^1.4.17", - "source-map-support": "^0.5.10", "tape": "^4.10.1", - "uglifyjs-webpack-plugin": "^1.2.7", - "webpack": "^4.29.5", + "terser-webpack-plugin": "^4.2.3", + "webpack": "^4.29", "webpack-command": "^0.4.2" }, "dependencies": { "@solgenomics/brapijs": "github:solgenomics/brapi-js#develop", "BrAPI-BoxPlotter": "git+https://github.com/solgenomics/BrAPI-BoxPlotter.git", + "core-js": "^3", "d3": "^7.3.0", "d3-array": "^2.11.0", "d3-path": "^1.0.9", "d3-sankey": "^0.12.3", "d3-shape": "^1.3.7", - "internmap": "^1.0.0" + "internmap": "^1.0.0", + "regenerator": "latest" }, "engines": { "node": ">=10.6.0", diff --git a/js/source/entries/boxplotter.js b/js/source/entries/boxplotter.js index 88fbd8ac2c..6f3b2946f8 100644 --- a/js/source/entries/boxplotter.js +++ b/js/source/entries/boxplotter.js @@ -1,14 +1,16 @@ -import BrAPIBoxPlotter from 'BrAPI-BoxPlotter'; -import '../legacy/d3/d3v4Min.js'; -import '../legacy/jquery.js'; -import '../legacy/brapi/BrAPI.js'; +import BrAPIBoxPlotter from "BrAPI-BoxPlotter"; +import "../legacy/d3/d3v4Min.js"; +import "../legacy/jquery.js"; +import "../legacy/brapi/BrAPI.js"; -export function init(main_div){ - if (!(main_div instanceof HTMLElement)){ - main_div = document.getElementById(main_div.startsWith("#") ? main_div.slice(1) : main_div); - } - - main_div.innerHTML = ` +export function init(main_div) { + if (!(main_div instanceof HTMLElement)) { + main_div = document.getElementById( + main_div.startsWith("#") ? main_div.slice(1) : main_div + ); + } + + main_div.innerHTML = `
    @@ -36,108 +38,128 @@ export function init(main_div){
    `; - - var bp = $(main_div); - var boxplot = BrAPIBoxPlotter(bp.find(".boxplotter-result").get(0)); - - - function loadDatasetObsUnits(ds,ou,auth_token){ - var d = { - 'dataset':ds - }; - console.log(d); - bp.find(".boxplotter-variable-select-div, .boxplotter-group-list, .boxplotter-result").hide(); - bp.find(".boxplotter-loading").html('Loading ...'); - $.ajax({ - url : document.location.origin+'/ajax/tools/boxplotter/get_constraints', - type : 'GET', - data : d, - dataType:'json', - success : function(data) { - //console.log(data); - var obsUnits = BrAPI(document.location.origin+"/brapi/v2","",auth_token).search_observationunits({ - "germplasmDbIds" : data["categories"]["accessions"], - "observationVariableDbIds" : data["categories"]["traits"], - "studyDbIds" : data["categories"]["trials"], - "locationDbIds" : data["categories"]["locations"], - "programDbIds" : data["categories"]["breeding_programs"], - "observationLevelName" : ou, - "includeObservations" : "true", - "pageSize" : 100000 - }) - boxplot.setData(obsUnits); - obsUnits.all(function(d){ - console.log(d); + + var bp = $(main_div); + var boxplot = BrAPIBoxPlotter(bp.find(".boxplotter-result").get(0)); + + function loadDatasetObsUnits(ds, ou, auth_token) { + var d = { + dataset: ds, + }; + console.log(d); + bp.find( + ".boxplotter-variable-select-div, .boxplotter-group-list, .boxplotter-result" + ).hide(); + bp.find(".boxplotter-loading").html("Loading ..."); + $.ajax({ + url: document.location.origin + "/ajax/tools/boxplotter/get_constraints", + type: "GET", + data: d, + dataType: "json", + success: function (data) { + //console.log(data); + var obsUnits = BrAPI( + document.location.origin + "/brapi/v2", + "", + auth_token + ).search_observationunits({ + germplasmDbIds: data["categories"]["accessions"], + observationVariableDbIds: data["categories"]["traits"], + studyDbIds: data["categories"]["trials"], + locationDbIds: data["categories"]["locations"], + programDbIds: data["categories"]["breeding_programs"], + observationLevelName: ou, + includeObservations: "true", + pageSize: 100000, + }); + boxplot.setData(obsUnits); + obsUnits.all(function (d) { + console.log(d); + }); + drawVariableSelect(); + drawGroupBys(); + boxplot.getVariables().then((vs) => { + if (vs.length < 1) { + bp.find(".boxplotter-result").html( + 'Not Enough Data with the Given Constraints' + ); + bp.find(".boxplots").hide(); + } else { + readGrouping(); + bp.find(".boxplotter-result .not-enough-data").remove(); + bp.find( + ".boxplotter-variable-select-div, .boxplotter-group-list, .brapp-wrapper, .boxplots" + ).show(); + } + bp.find(".boxplotter-loading").html(""); + bp.find(".boxplotter-result").show(); + }); + }, + }); + } + function drawVariableSelect() { + boxplot.getVariables().then((vs) => { + var vars = d3 + .select(main_div) + .select(".boxplotter-variable-select") + .selectAll("option") + .data(vs); + vars.exit().remove(); + var allVars = vars + .enter() + .append("option") + .merge(vars) + .attr("value", (d) => d.key) + .attr("selected", (d) => (d.key == boxplot.variable ? "" : null)) + .text((d) => d.value); }); - drawVariableSelect(); - drawGroupBys(); - boxplot.getVariables().then(vs=>{ - if(vs.length<1){ - bp.find('.boxplotter-result').html('Not Enough Data with the Given Constraints'); - bp.find(".boxplots").hide(); - } - else{ - readGrouping(); - bp.find('.boxplotter-result .not-enough-data').remove(); - bp.find(".boxplotter-variable-select-div, .boxplotter-group-list, .brapp-wrapper, .boxplots").show(); - } - bp.find(".boxplotter-loading").html(''); - bp.find(".boxplotter-result").show(); + d3.select(main_div) + .select(".boxplotter-variable-select") + .on("change", function () { + boxplot.setVariable(this.value); + }); + } - }) - } - }); - } - function drawVariableSelect(){ - boxplot.getVariables().then(vs=>{ - var vars = d3.select(main_div).select(".boxplotter-variable-select").selectAll("option").data(vs); - vars.exit().remove(); - var allVars = vars.enter().append("option") - .merge(vars) - .attr("value",d=>d.key) - .attr("selected",d=>d.key==boxplot.variable?"":null) - .text(d=>d.value); - }); - d3.select(main_div).select(".boxplotter-variable-select").on("change",function(){ - boxplot.setVariable(this.value); - }) - } - - function drawGroupBys(){ - boxplot.getGroupings().then(grps=>{ - console.log("grps",grps); - var optSelects = d3.select(main_div).select(".boxplotter-group-list") - .selectAll(".groupBy") - .on("change",function(){ - readGrouping(); - }) - .selectAll('option:not([value=""])') - .data(d=>grps); - optSelects.enter().append("option") - .merge(optSelects) - .attr("value",d=>d.key) - .text(d=>d.value.name); - d3.selectAll(".groupBy-add").on("click",function(){ - $(this.parentNode).clone(true).appendTo(this.parentNode.parentNode); - drawGroupBys(); - readGrouping(); - }); - d3.selectAll(".groupBy-remove").on("click",function(){ - d3.select(this.parentNode).remove(); - readGrouping(); - }); - }); - } - function readGrouping(){ - var grouping = []; - d3.selectAll(".groupBy") - .each(function(){grouping.push(this.value)}) - boxplot.setGroupings(grouping) - } - - return { - 'loadDatasetObsUnits':loadDatasetObsUnits, - 'boxplot':boxplot, - 'element':main_div - } + function drawGroupBys() { + boxplot.getGroupings().then((grps) => { + console.log("grps", grps); + var optSelects = d3 + .select(main_div) + .select(".boxplotter-group-list") + .selectAll(".groupBy") + .on("change", function () { + readGrouping(); + }) + .selectAll('option:not([value=""])') + .data((d) => grps); + optSelects + .enter() + .append("option") + .merge(optSelects) + .attr("value", (d) => d.key) + .text((d) => d.value.name); + d3.selectAll(".groupBy-add").on("click", function () { + $(this.parentNode).clone(true).appendTo(this.parentNode.parentNode); + drawGroupBys(); + readGrouping(); + }); + d3.selectAll(".groupBy-remove").on("click", function () { + d3.select(this.parentNode).remove(); + readGrouping(); + }); + }); + } + function readGrouping() { + var grouping = []; + d3.selectAll(".groupBy").each(function () { + grouping.push(this.value); + }); + boxplot.setGroupings(grouping); + } + + return { + loadDatasetObsUnits: loadDatasetObsUnits, + boxplot: boxplot, + element: main_div, + }; } diff --git a/js/source/entries/dataset_scatterplot.js b/js/source/entries/dataset_scatterplot.js index fb45fb8fcb..220d9f315a 100644 --- a/js/source/entries/dataset_scatterplot.js +++ b/js/source/entries/dataset_scatterplot.js @@ -1,136 +1,141 @@ export function init(datasetId, datasetName) { - class Dataset { - constructor() { - this.datasets = {}; - this.data = []; - // store phenotype id, trait, value - this.outliers = []; - this.outlierCutoffs = new Set(); - this.firstRefresh = true; - this.stdDevMultiplier = $("#outliers_range").slider("value"); - this.selection = "default"; - this.datasetId = datasetId; - this.observations = {}; - this.traits = {}; - this.traitsIds = {}; - this.phenoIds = []; - this.traitVals = []; - this.storedOutliersIds = []; - this.metricValue = document.querySelector('input[name="dataset_metric"]:checked').value; - } - - getPhenotypes() { - const LocalThis = this; - this.firstRefresh = false; - new jQuery.ajax({ - url: '/ajax/dataset/retrieve/' + this.datasetId + '/phenotypes?include_phenotype_primary_key=1', - success: function(response) { - LocalThis.observations = response.phenotypes; - LocalThis.setDropDownTraits(); - }, - error: function(response) { - alert('Error'); - } - }) - } - - getTraits() { + class Dataset { + constructor() { + this.datasets = {}; + this.data = []; + // store phenotype id, trait, value + this.outliers = []; + this.outlierCutoffs = new Set(); + this.firstRefresh = true; + this.stdDevMultiplier = $("#outliers_range").slider("value"); + this.selection = "default"; + this.datasetId = datasetId; + this.observations = {}; + this.traits = {}; + this.traitsIds = {}; + this.phenoIds = []; + this.traitVals = []; + this.storedOutliersIds = []; + this.metricValue = document.querySelector('input[name="dataset_metric"]:checked').value; + } + + getPhenotypes() { const LocalThis = this; this.firstRefresh = false; new jQuery.ajax({ - url: '/ajax/dataset/retrieve/' + this.datasetId + '/traits', - success: function(response) { - LocalThis.traitsIds = response.traits.map( - trait => trait[0] - ); - }, - error: function(response) { - alert('Error'); - } + url: '/ajax/dataset/retrieve/' + this.datasetId + '/phenotypes?include_phenotype_primary_key=1', + success: function (response) { + LocalThis.observations = response.phenotypes; + LocalThis.setDropDownTraits(); + }, + error: function (response) { + alert('Error'); + } }) - } + } + + getTraits() { + const LocalThis = this; + this.firstRefresh = false; + new jQuery.ajax({ + url: '/ajax/dataset/retrieve/' + this.datasetId + '/traits', + success: function (response) { + LocalThis.traitsIds = response.traits.map( + trait => trait[0] + ); + // console.log(LocalThis.traitsIds); + }, + error: function (response) { + alert('Error'); + } + }) + } getStoredOutliers() { - const LocalThis = this; + const LocalThis = this; new jQuery.ajax({ type: 'POST', url: '/ajax/dataset/retrieve_outliers/' + LocalThis.datasetId, - success: function(response) { + success: function (response) { LocalThis.storedOutliersIds = response.outliers !== null ? response.outliers : []; }, - error: function(response) { + error: function (response) { alert('Error'); } }) } - setDropDownTraits() { - const keys = this.observations[0]; - // Construct trait object - for (let i = 39; i < keys.length - 1; i++) { + setDropDownTraits() { + const keys = this.observations[0]; + // Construct trait object + for (let i = 39; i < keys.length - 1; i++) { if (i % 2 == 1 && i <= keys.length - 2) { - this.traits[keys[i]] = {}; - } - } + this.traits[keys[i]] = {}; + } + } - for (let i = 1; i < this.observations.length; i++) { + for (let i = 1; i < this.observations.length; i++) { // Goes through each observation, and populates the traits hash with each trait, using the phenotype id as the key, and the traitValue as the value. - for (let j = 39; j < this.observations[i].length - 1; j++) { + for (let j = 39; j < this.observations[i].length - 1; j++) { if (j % 2 == 1) { - this.traits[keys[j]][this.observations[i][j+1]] = this.observations[i][j]; + this.traits[keys[j]][this.observations[i][j + 1]] = this.observations[i][j]; } - } - } - // Use traits to set select options - const select = document.getElementById("trait_selection"); - for (const traitName of Object.keys(this.traits)) { + } + } + // Use traits to set select options + const select = document.getElementById("trait_selection"); + for (const traitName of Object.keys(this.traits)) { const option = document.createElement("option"); option.value = traitName; option.innerHTML = traitName; select.appendChild(option); - } - - } - - setData() { - if (this.selection != "default") { - // Gets a list of pheno ids, filters them so only the ones that have non null values are included and then sorts the ids by their value by looking up their values in traits hash. - this.phenoIds = Object.keys(this.traits[this.selection]) - .filter((phenoId) => !isNaN(parseFloat(this.traits[this.selection][phenoId]))) - .sort((a,b) => this.traits[this.selection][a] - this.traits[this.selection][b]); - - this.traitVals = this.phenoIds.map((id) => parseFloat(this.traits[this.selection][id])); - // console.log(this.traitVals); - // Debugging check: You should see a list of ids and the corresponding values, logs should be sorted by increasing values. - // for (let id of this.phenoIds) { - // console.log(id, this.traits[this.selection][id].value); - // } - } - } - - standardDeviation(values) { - var average = function(data) { - var sum = data.reduce(function(sum, value){ - return sum + value; - }, 0); - - var avg = sum / data.length; - return avg; - } - - var avg = average(values); - - var squareDiffs = values.map(function(value){ - var diff = value - avg; - var sqrDiff = diff * diff; - return sqrDiff; - }); - - var avgSquareDiff = average(squareDiffs); - - var stdDev = Math.sqrt(avgSquareDiff); - return [avg, stdDev]; - } + } + + } + + setData() { + if (this.selection != "default") { + // Gets a list of pheno ids, filters them so only the ones that have non null values are included and then sorts the ids by their value by looking up their values in traits hash. + + // console.log(this.traits); + this.phenoIds = Object.keys(this.traits[this.selection]) + .filter((phenoId) => !isNaN(parseFloat(this.traits[this.selection][phenoId]))) + .sort((a, b) => this.traits[this.selection][a] - this.traits[this.selection][b]); + + // console.log(this.phenoIds); + + this.traitVals = this.phenoIds.map((id) => parseFloat(this.traits[this.selection][id])); + // console.log(this.traitVals); + // Debugging check: You should see a list of ids and the corresponding values, logs should be sorted by increasing values. + // for (let id of this.phenoIds) { + // console.log(id, this.traits[this.selection][id].value); + // } + } + } + + standardDeviation(values) { + var average = function (data) { + var sum = data.reduce(function (sum, value) { + return sum + value; + }, 0); + + var avg = sum / data.length; + return avg; + } + + var avg = average(values); + + var squareDiffs = values.map(function (value) { + var diff = value - avg; + var sqrDiff = diff * diff; + return sqrDiff; + }); + + var avgSquareDiff = average(squareDiffs); + + var stdDev = Math.sqrt(avgSquareDiff); + return [avg, stdDev]; + } median(values) { if (!Array.isArray(values)) return 0; @@ -152,166 +157,314 @@ export function init(datasetId, datasetName) { return this.median(medianMap); } // OK - tested - - addEventListeners() { - let LocalThis = this; - - // Handle Slider Events - var sliderSelector = $("#outliers_range"); - sliderSelector.on("slidechange", (event, ui) => { - LocalThis.stdDevMultiplier = ui.value; + + + quartile(values, factor) { + if (!Array.isArray(values)) return 0; + + let sorted = [...values].sort((a, b) => a - b); + let index = (sorted.length - 1) * factor; + let lowerIndex = Math.floor(index); + let upperIndex = Math.ceil(index); + let interpolation = index - lowerIndex; + return sorted[lowerIndex] * (1 - interpolation) + sorted[upperIndex] * interpolation; + } + + + iqr(values) { + let q1 = this.quartile(values, 0.25); + let q3 = this.quartile(values, 0.75); + return q3 - q1; + } + + addEventListeners() { + let LocalThis = this; + + // Handle Slider Events + var sliderSelector = $("#outliers_range"); + sliderSelector.on("slidechange", (event, ui) => { + LocalThis.stdDevMultiplier = ui.value; LocalThis.outliers = []; LocalThis.outlierCutoffs = new Set(); d3.select("svg").remove(); LocalThis.render(); - }) - // Handle Metric Radio - var metricSelectors = document.querySelectorAll('input[type=radio][name="dataset_metric"]'); - - Array.prototype.forEach.call(metricSelectors, (metricRadio) => { - metricRadio.addEventListener("change", (event) => { - - - LocalThis.metricValue = document.querySelector('input[name="dataset_metric"]:checked').value; + }) + // Handle Metric Radio + var metricSelectors = document.querySelectorAll('input[type=radio][name="dataset_metric"]'); + + Array.prototype.forEach.call(metricSelectors, (metricRadio) => { + metricRadio.addEventListener("change", (event) => { + + + LocalThis.metricValue = document.querySelector('input[name="dataset_metric"]:checked').value; d3.select("svg").remove(); LocalThis.render(); - }) - }) + }) + }) - // Handle Select Events - let selection = document.getElementById("trait_selection"); - selection.addEventListener("change", (event) => { + // Handle Select Events + let selection = document.getElementById("trait_selection"); + selection.addEventListener("change", (event) => { d3.select("svg").remove(); LocalThis.selection = event.target.value; LocalThis.setData(); LocalThis.outliers = []; LocalThis.outlierCutoffs = new Set(); - sliderSelector.slider("option", "value", 3); - LocalThis.stdDevMultiplier = sliderSelector.slider("value"); - if (!this.firstRefresh) { - d3.select("svg").remove(); - LocalThis.render(); - } - }); - - let storeOutliersButton = document.getElementById("store_outliers"); - storeOutliersButton.onclick = function() { - let allOutliers = new Set(LocalThis.storedOutliersIds.concat(LocalThis.outliers)); - let stringOutliers = [...allOutliers].join(','); - let stringOutlierCutoffs = [...LocalThis.outlierCutoffs].join(','); - new jQuery.ajax({ + sliderSelector.slider("option", "value", 3); + LocalThis.stdDevMultiplier = sliderSelector.slider("value"); + if (!this.firstRefresh) { + d3.select("svg").remove(); + LocalThis.render(); + removeRosnserTable(); + } + }); + + let storeOutliersButton = document.getElementById("store_outliers"); + storeOutliersButton.onclick = function () { + let allOutliers = new Set(LocalThis.storedOutliersIds.concat(LocalThis.outliers)); + let stringOutliers = [...allOutliers].join(','); + let stringOutlierCutoffs = [...LocalThis.outlierCutoffs].join(','); + new jQuery.ajax({ type: 'POST', url: '/ajax/dataset/store_outliers/' + LocalThis.datasetId, - data: {outliers: stringOutliers, outlier_cutoffs: stringOutlierCutoffs }, - success: function(response) { - alert('outliers successfully stored!'); + data: { outliers: stringOutliers, outlier_cutoffs: stringOutlierCutoffs }, + success: function (response) { + alert('outliers successfully stored!'); LocalThis.storedOutliersIds = [...allOutliers]; - d3.select("svg").remove(); - LocalThis.render(); + d3.select("svg").remove(); + LocalThis.render(); }, - error: function(response) { - alert('Error'); - } - }) - } + error: function (response) { + alert('Error'); + } + }) + } let resetOutliersButton = document.getElementById("reset_outliers"); - resetOutliersButton.onclick = function() { + resetOutliersButton.onclick = function () { new jQuery.ajax({ type: 'POST', url: '/ajax/dataset/store_outliers/' + LocalThis.datasetId, - data: {outliers: "", outlier_cutoffs: "" }, - success: function(response) { + data: { outliers: "", outlier_cutoffs: "" }, + success: function (response) { alert('outliers successfully reseted!'); - LocalThis.storedOutliersIds = []; - d3.select("svg").remove(); - LocalThis.render(); + LocalThis.storedOutliersIds = []; + d3.select("svg").remove(); + LocalThis.render(); + removeRosnserTable(); }, - error: function(response) { + error: function (response) { alert('Error'); } }) } let resetTraitOutliersButton = document.getElementById("reset_trait"); - resetTraitOutliersButton.onclick = function() { + resetTraitOutliersButton.onclick = function () { let filteredNonTrait = LocalThis.storedOutliersIds.filter(elem => !LocalThis.phenoIds.includes(elem)); - let stringFilteredNonTrait = filteredNonTrait.join(','); - new jQuery.ajax({ + let stringFilteredNonTrait = filteredNonTrait.join(','); + new jQuery.ajax({ type: 'POST', url: '/ajax/dataset/store_outliers/' + LocalThis.datasetId, - data: {outliers: stringFilteredNonTrait, outlier_cutoffs: "" }, - success: function(response) { + data: { outliers: stringFilteredNonTrait, outlier_cutoffs: "" }, + success: function (response) { alert('outliers successfully reseted!'); - LocalThis.storedOutliersIds = filteredNonTrait; - d3.select("svg").remove(); - LocalThis.render(); + LocalThis.storedOutliersIds = filteredNonTrait; + d3.select("svg").remove(); + LocalThis.render(); + removeRosnserTable(); }, - error: function(response) { - alert('Error'); + error: function (response) { + alert('Error'); } - }) + }) } - } + let rosnersTestButton = document.getElementById("rosner_test"); + rosnersTestButton.onclick = function () { + if (document.getElementById("trait_selection").value == 'default') return; + // add spinner + document.getElementById("loading-spinner").style.visibility = 'visible'; + new jQuery.ajax({ + type: 'POST', + url: '/ajax/dataset/rosner_test/' + LocalThis.datasetId, + data: { dataset_trait: document.getElementById("trait_selection").value }, + success: function (response) { + // alert(response.message); + createOutlierTable(response.file); + document.getElementById("loading-spinner").style.visibility = 'hidden'; + }, + error: function (response) { + alert('Error'); + document.getElementById("loading-spinner").style.visibility = 'visible'; + } + }) + } + + function createOutlierTable(file) { + // check if exist and remove + let data_rows = file.slice(1, file.length); + let column_names = file[0].map(value => ({ "title": value })); + let outliers = data_rows.filter((elem) => { return elem[7] == 'TRUE' }).map((elem) => elem[4]); + + removeRosnserTable(); + + let table = document.createElement("table"); + table.setAttribute("id", "rosner_table"); + table.classList = "display"; + document.getElementById("statistic_tests").appendChild(table); + + let addRosnerButton = document.createElement("button"); + if (outliers.length == 0) { + addRosnerButton.disabled = true; + } + addRosnerButton.setAttribute("id", "rosner_add"); + addRosnerButton.classList = "btn btn-sm btn-success btn-dataset"; + addRosnerButton.textContent = "Add Rosner test outliers"; + addRosnerButton.addEventListener('click', function () { + addRosnserOutliers(outliers); + }); + + document.getElementById("statistic_tests").appendChild(addRosnerButton); + + jQuery('#rosner_table').DataTable({ + columns: column_names, + data: data_rows, + searching: false, + paging: false, + info: false + }); + + return false; + } + + function addRosnserOutliers(rosnerOutliers) { + if (rosnerOutliers.length == 0) { + return; + } + // console.log(rosnerOutliers); + + let allOutliers = new Set(LocalThis.storedOutliersIds.concat(rosnerOutliers)); + let stringOutliers = [...allOutliers].join(','); + let stringOutlierCutoffs = [...LocalThis.outlierCutoffs].join(','); + new jQuery.ajax({ + type: 'POST', + url: '/ajax/dataset/store_outliers/' + LocalThis.datasetId, + data: { outliers: stringOutliers, outlier_cutoffs: stringOutlierCutoffs }, + success: function (response) { + alert('outliers successfully stored!'); + LocalThis.storedOutliersIds = [...allOutliers]; + d3.select("svg").remove(); + LocalThis.render(); + }, + error: function (response) { + alert('Error'); + } + }) + + return false; + } + + function removeRosnserTable() { + if ($.fn.DataTable.isDataTable("#rosner_table")) { + jQuery('#rosner_table').DataTable().destroy(); + } + if (document.getElementById("rosner_table")) { + document.getElementById("rosner_table").remove(); + document.getElementById("rosner_add").remove(); + } + } + } + + + + render() { + if (this.firstRefresh) { + this.getPhenotypes(); + this.getTraits(); + this.getStoredOutliers(); + this.addEventListeners(); + } else if (this.selection != "default") { + + const LocalThis = this; + + const [mean, stdDev] = this.standardDeviation(this.traitVals); + const [median, mad] = [this.median(this.traitVals), this.mad(this.traitVals)]; + const [quartiles, factor] = [[this.quartile(this.traitVals, 0.25), this.quartile(this.traitVals, 0.75)], this.iqr(this.traitVals)]; + + let settings; + switch (LocalThis.metricValue) { + case "mean": + settings = [mean, stdDev, "Mean", "Std Dev"]; + break; + case "median": + settings = [median, mad, "Median", "MAD"]; + break; + case "iqr": + settings = [quartiles, factor, "Q1,Q3", "IQR"]; + break; + default: + settings = [median, mad, "Median", "MAD"]; + } + const [metric, deviation, metricString, deviationString] = settings; - render() { - if (this.firstRefresh) { - this.getPhenotypes(); - this.getTraits(); - this.getStoredOutliers(); - this.addEventListeners(); - } else if (this.selection != "default") { + let filter = LocalThis.stdDevMultiplier; - const LocalThis = this; + function rightCutoffCalc() { + if (LocalThis.metricValue == "iqr") { + return quartiles[1] + factor * filter; + } else { + return metric + deviation * filter; + } + } - const [mean, stdDev] = this.standardDeviation(this.traitVals); - const [median, mad]= [this.median(this.traitVals), this.mad(this.traitVals)]; - const [metric, deviation, metricString, deviationString] = LocalThis.metricValue === "mean" ? [mean, stdDev, "Mean", "Std Dev"] : [median, mad, "Median", "MAD"]; + function leftCutoffCalc() { + if (LocalThis.metricValue == "iqr") { + return quartiles[0] - factor * filter; + } else { + return Math.max(metric - deviation * filter, 0); + } + } - let filter = LocalThis.stdDevMultiplier ; - let rightCutoff = metric + deviation * filter; - let leftCutoff = Math.max(metric - deviation * filter, 0); + let rightCutoff = rightCutoffCalc(); + let leftCutoff = leftCutoffCalc(); - this.outliers = []; + this.outliers = []; - const margin = {top: 10, right: 30, bottom: 30, left: 60}, + const margin = { top: 10, right: 30, bottom: 30, left: 60 }, width = 1180 - margin.left - margin.right, - height = 600 - margin.top - margin.bottom; + height = 600 - margin.top - margin.bottom; - var svg = d3.select("#trait_graph") + var svg = d3.select("#trait_graph") .append("svg") - .attr("width", width + margin.left + margin.right) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr( - "transform", - "translate(" + margin.left + "," + margin.top + ")" - ); + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr( + "transform", + "translate(" + margin.left + "," + margin.top + ")" + ); const greenColor = "#00ba38" , yellowColor = "#ffe531" , redColor = "#f7756c" - , blueColor = "#337ab7"; - - var isOutlier = function(id, value, metric, deviation) { - let filter = (LocalThis.stdDevMultiplier); - let leftCutoff = Math.max(metric - deviation * filter, 0); - let rightCutoff = metric + deviation * filter; // here max from scale - let color = ""; - let stroke; - - - if (value >= leftCutoff && value <= rightCutoff) { - color = greenColor; + , blueColor = "#337ab7"; + + var isOutlier = function (id, value, leftCutoff, rightCutoff) { + + let color = ""; + let stroke; + + if (value >= leftCutoff && value <= rightCutoff) { + color = greenColor; } - if (LocalThis.storedOutliersIds.includes(id.toString())) { - stroke = "black"; + if (LocalThis.storedOutliersIds.includes(id.toString())) { + stroke = "black"; } - if (value <= leftCutoff || value >= rightCutoff){ - if (leftCutoff >= 0) {LocalThis.outlierCutoffs.add(leftCutoff)}; + if (value <= leftCutoff || value >= rightCutoff) { + if (leftCutoff >= 0) { LocalThis.outlierCutoffs.add(leftCutoff) }; LocalThis.outlierCutoffs.add(rightCutoff); LocalThis.outliers.push(id); color = redColor; @@ -319,176 +472,183 @@ export function init(datasetId, datasetName) { return [color, stroke]; } - - // Add ackground ggplot2 like + + // Add ackground ggplot2 like svg.append("rect") - .attr("x",0) - .attr("y",0) + .attr("x", 0) + .attr("y", 0) .attr("height", height) .attr("width", width) .style("fill", "#ebebeb") - // Add X axis - var x = d3.scaleLinear() - .domain([0, this.phenoIds.length]) - .range([ 0, width]); + // Add X axis + var x = d3.scaleLinear() + .domain([0, this.phenoIds.length]) + .range([0, width]); - // Add Y axis - var y = d3.scaleLinear() - .domain([Math.min(...this.traitVals), Math.max(...this.traitVals)]) - .range([ height, 0]); + // Add Y axis + var y = d3.scaleLinear() + .domain([Math.min(...this.traitVals), Math.max(...this.traitVals)]) + .range([height, 0]); svg.append("g") - .call(d3.axisLeft(y).tickSize(-width*1).ticks(10)) - .style("font", "16px arial") + .call(d3.axisLeft(y).tickSize(-width * 1).ticks(10)) + .style("font", "16px arial") // Customization svg.selectAll(".tick line").attr("stroke", "white") - svg.append("text") - .attr("transform", "rotate(-90)") - .attr("y", 0 - margin.left) - .attr("x",0 - (height / 2)) + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", 0 - margin.left) + .attr("x", 0 - (height / 2)) .attr("dy", ".65em") - .style("text-anchor", "middle") - .style("font", "16px arial") - .text("Trait Value"); - - - // Add dots - var tooltip = d3.select("#trait_graph") - .append("div") - .attr("id", "tooltip") - .attr("class", "tooltip") - .style("background-color", "white") - .style("border", "solid") - .style("border-width", "2px") - .style("font-size", "15px") - .style("border-radius", "5px") - .style("padding", "5px") - .style("opacity", 0); - - var mouseover = function(d) { - tooltip - .style("opacity", 1) - d3.select(this) - .style("fill", "white") - .style("opacity", 1) - } - - var mousemove = function(d) { - tooltip + .style("text-anchor", "middle") + .style("font", "16px arial") + .text("Trait Value"); + + + // Add dots + var tooltip = d3.select("#trait_graph") + .append("div") + .attr("id", "tooltip") + .attr("class", "tooltip") + .style("background-color", "white") + .style("border", "solid") + .style("border-width", "2px") + .style("font-size", "15px") + .style("border-radius", "5px") + .style("padding", "5px") + .style("opacity", 0); + + var mouseover = function (d) { + tooltip + .style("opacity", 1) + d3.select(this) + .style("fill", "white") + .style("opacity", 1) + } + + var mousemove = function (d) { + tooltip .html("id: " + LocalThis.phenoIds[d] + "
    " + "val: " + LocalThis.traitVals[d]) - .style("left", (d3.mouse(this)[0]+50) + "px") - .style("top", (d3.mouse(this)[1]+40) + "px") - } - - var mouseleave = function(d) { - tooltip - .style("opacity", 0) - d3.select(this) - .style("fill", function(d) {return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], metric, deviation)[0]}) - .style("stroke-width", 2) - .style("stroke", function(d) {return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], metric, deviation)[1]}) - .style("fill-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.2 : 0.8) }) - .style("stroke-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.4 : 0.8) }) - } - - svg.append('g') - .selectAll("dot") - .data([...Array(this.phenoIds.length).keys()]) - .enter() - .append("circle") - .attr("cx", function (d) { return x(d); } ) - .attr("cy", function (d) { return y(LocalThis.traitVals[d]); } ) - .attr("r", 6) - .style("fill", function(d) {return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], metric, deviation)[0]}) - .style("stroke-width", 2) - .style("stroke", function(d) {return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], metric, deviation)[1]}) + .style("left", (d3.mouse(this)[0] + 50) + "px") + .style("top", (d3.mouse(this)[1] + 40) + "px") + } + + var mouseleave = function (d) { + tooltip + .style("opacity", 0) + d3.select(this) + .style("fill", function (d) { return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], leftCutoff, rightCutoff)[0] }) + .style("stroke-width", 2) + .style("stroke", function (d) { return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], leftCutoff, rightCutoff)[1] }) .style("fill-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.2 : 0.8) }) - .style("stroke-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.4 : 0.8) }) - .on("mouseover", mouseover) - .on("mousemove", mousemove) - .on("mouseleave", mouseleave); + .style("stroke-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.4 : 0.8) }) + } + + svg.append('g') + .selectAll("dot") + .data([...Array(this.phenoIds.length).keys()]) + .enter() + .append("circle") + .attr("cx", function (d) { return x(d); }) + .attr("cy", function (d) { return y(LocalThis.traitVals[d]); }) + .attr("r", 6) + .style("fill", function (d) { return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], leftCutoff, rightCutoff)[0] }) + .style("stroke-width", 2) + .style("stroke", function (d) { return isOutlier(LocalThis.phenoIds[d], LocalThis.traitVals[d], leftCutoff, rightCutoff)[1] }) + .style("fill-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.2 : 0.8) }) + .style("stroke-opacity", (d) => { return (LocalThis.traitVals[d] <= leftCutoff || LocalThis.traitVals[d] >= rightCutoff ? 0.4 : 0.8) }) + .on("mouseover", mouseover) + .on("mousemove", mousemove) + .on("mouseleave", mouseleave); + + let metricArray; + if (LocalThis.metricValue != "iqr") { + metricArray = [metric]; + } else { + metricArray = metric; + } + + metricArray.forEach((number) => { + svg.append("line") + .attr("class", "mean-line") + .attr("x1", 0) + .attr("y1", y(number)) + .attr("x2", width) + .attr("y2", y(number)) + .attr("fill", "none") + .attr("stroke", "black"); + }) + + svg.append("line") + .attr("class", "sd-line-top") + .attr("x1", 0) + .attr("y1", y(rightCutoff)) + .attr("x2", width) + .attr("y2", y(rightCutoff)) + .attr("fill", "none") + .attr("stroke", "darkgrey"); svg.append("line") - .attr("class", "mean-line") - .attr("x1", 0) - .attr("y1", y(metric)) - .attr("x2", width) - .attr("y2", y(metric)) - .attr("fill", "none") - .attr("stroke", "black"); - - svg.append("line") - .attr("class", "sd-line-top") - .attr("x1", 0) - .attr("y1", y(rightCutoff)) - .attr("x2", width) - .attr("y2", y(rightCutoff)) - .attr("fill", "none") - .attr("stroke", "darkgrey"); - - svg.append("line") - .attr("class", "sd-line-bottom") - .attr("x1", 0) - .attr("y1", y(leftCutoff >= 0 ? leftCutoff : 0)) - .attr("x2", width) - .attr("y2", y(leftCutoff >= 0 ? leftCutoff : 0)) - .attr("fill", "none") - .attr("stroke", "darkgrey"); - - - - // legend builder - const legendSize = { - width: 250, - height: 135, - get posX() { + .attr("class", "sd-line-bottom") + .attr("x1", 0) + .attr("y1", y(leftCutoff >= 0 ? leftCutoff : 0)) + .attr("x2", width) + .attr("y2", y(leftCutoff >= 0 ? leftCutoff : 0)) + .attr("fill", "none") + .attr("stroke", "darkgrey"); + + // legend builder + const legendSize = { + width: 250, + height: 135, + get posX() { return 5; - }, - get posY() { - return 15 - } - }; - const dotSize = 7 - - const legend = svg.append("g") - .attr('id', 'legend') - .attr('height', legendSize.height) - .attr('width', legendSize.width) - .attr('transform', 'translate(5, 5)'); - - legend.append('rect') - .attr('height', legendSize.height) - .attr('width', legendSize.width) - .attr('x', 0) - .attr('y', 0) - .attr('fill', 'white') - .style("stroke", "lightgrey") - .style("stroke-width", 3); - - legend.append('circle') - .attr('r', dotSize) - .attr('class', 'dot-legend') - .attr('fill', greenColor) - .attr('cx', legendSize.posX + 20) - .attr('cy', legendSize.posY + 5) - - legend.append('circle') - .attr('r', dotSize) - .attr('stroke', "black") - .style("stroke-width", 2) - .style("fill", "none") - .attr('class', 'dot-legend') - .attr('cx', legendSize.posX + 20) - .attr('cy', legendSize.posY + 30) - - legend.append('circle') - .attr('r', dotSize) - .attr('class', 'dot-legend') - .attr('fill', redColor) - .attr('cx', legendSize.posX + 20) - .attr('cy', legendSize.posY + 55) + }, + get posY() { + return 15 + } + }; + const dotSize = 7 + + const legend = svg.append("g") + .attr('id', 'legend') + .attr('height', legendSize.height) + .attr('width', legendSize.width) + .attr('transform', 'translate(5, 5)'); + + legend.append('rect') + .attr('height', legendSize.height) + .attr('width', legendSize.width) + .attr('x', 0) + .attr('y', 0) + .attr('fill', 'white') + .style("stroke", "lightgrey") + .style("stroke-width", 3); + + legend.append('circle') + .attr('r', dotSize) + .attr('class', 'dot-legend') + .attr('fill', greenColor) + .attr('cx', legendSize.posX + 20) + .attr('cy', legendSize.posY + 5) + + legend.append('circle') + .attr('r', dotSize) + .attr('stroke', "black") + .style("stroke-width", 2) + .style("fill", "none") + .attr('class', 'dot-legend') + .attr('cx', legendSize.posX + 20) + .attr('cy', legendSize.posY + 30) + + legend.append('circle') + .attr('r', dotSize) + .attr('class', 'dot-legend') + .attr('fill', redColor) + .attr('cx', legendSize.posX + 20) + .attr('cy', legendSize.posY + 55) svg.append('text') @@ -503,45 +663,42 @@ export function init(datasetId, datasetName) { .style("font", "arial") .text('outliers value stored in database') - svg.append('text') + svg.append('text') .attr('x', legendSize.posX + 25 + dotSize + 5) .attr('y', legendSize.posY + 60 + dotSize / 2 + 1) .style("font", "arial") .text('outliers from current cutoff') - legend.append("text") - .text(metricString + " : " + metric.toFixed(2)) - .attr('x', legendSize.posX + 20 - dotSize / 2) - .attr('y', legendSize.posY + 85); - - legend.append("text") - .text(deviationString + " : " + deviation.toFixed(2)) - .style("font", "arial") - .attr('x', legendSize.posX + 120 - dotSize / 2) - .attr('y', legendSize.posY + 85); - - legend.append("text") - .text(() => { - let leftCutoff = metric - deviation * LocalThis.stdDevMultiplier; - return "L. Cutoff: " + leftCutoff.toFixed(2); - }) - .style("font", "arial") - .attr('x', legendSize.posX + 20 - dotSize / 2) - .attr('y', legendSize.posY + 110); - - legend.append("text") - .text(() => { - let rightCutoff = metric + deviation * LocalThis.stdDevMultiplier; - return "R. Cutoff: " + rightCutoff.toFixed(2); - }) - .style("font", "arial") - .attr('x', legendSize.posX + 120 - dotSize / 2) - .attr('y', legendSize.posY + 110); - - } - } - } - const dataset = new Dataset; - return dataset; + legend.append("text") + .text(metricString + ": " + (LocalThis.metricValue == "iqr" ? "(" + metric[0].toFixed(1) + ", " + metric[1].toFixed(1) + ")" : metric.toFixed(2))) + .attr('x', legendSize.posX + 20 - dotSize / 2) + .attr('y', legendSize.posY + 85); + + legend.append("text") + .text(deviationString + ": " + deviation.toFixed(2)) + .style("font", "arial") + .attr('x', legendSize.posX + (LocalThis.metricValue == "iqr" ? 165 : 130) - dotSize / 2) + .attr('y', legendSize.posY + 85); + + legend.append("text") + .text(() => { + return "L. Cutoff: " + leftCutoff.toFixed(2); + }) + .style("font", "arial") + .attr('x', legendSize.posX + 20 - dotSize / 2) + .attr('y', legendSize.posY + 110); + + legend.append("text") + .text(() => { + return "R. Cutoff: " + rightCutoff.toFixed(2); + }) + .style("font", "arial") + .attr('x', legendSize.posX + 130 - dotSize / 2) + .attr('y', legendSize.posY + 110); + } + } + } + const dataset = new Dataset; + return dataset; } \ No newline at end of file diff --git a/js/source/entries/fieldmap.js b/js/source/entries/fieldmap.js index 70ee521751..a3aa226dd8 100644 --- a/js/source/entries/fieldmap.js +++ b/js/source/entries/fieldmap.js @@ -1,10 +1,40 @@ -import '../legacy/d3/d3Min.js'; -import '../legacy/jquery.js'; -import '../legacy/brapi/BrAPI.js'; +import "../legacy/d3/d3v4Min.js"; +import "../legacy/jquery.js"; +import "../legacy/brapi/BrAPI.js"; +import { html, randomExponential } from "d3"; // Colors to use when labelling multiple trials -const trial_colors = ['#2f4f4f', '#ff8c00', '#ffff00', '#00ff00', '#9400d3', '#00ffff', '#1e90ff', '#ff1493', '#ffdab9', '#228b22']; -const trial_colors_text = ['#ffffff', '#000000', '#000000', '#000000', '#ffffff', '#000000', '#ffffff', '#ffffff', '#000000', '#ffffff']; +const trial_colors = [ + "#2f4f4f", + "#ff8c00", + "#ffff00", + "#00ff00", + "#9400d3", + "#00ffff", + "#1e90ff", + "#ff1493", + "#ffdab9", + "#228b22", +]; +const trial_colors_text = [ + "#ffffff", + "#000000", + "#000000", + "#000000", + "#ffffff", + "#000000", + "#ffffff", + "#ffffff", + "#000000", + "#ffffff", +]; + +var colors = [ + "white", + "darkred", +]; + +var colorScale; export function init() { class FieldMap { @@ -14,9 +44,9 @@ export function init() { this.plot_object = Object; this.meta_data = {}; this.brapi_plots = Object; - this.heatmap_queried = false; this.heatmap_selected = false; this.heatmap_selection = String; + this.heatmap_cached_data = {}; this.heatmap_object = Object; this.display_borders = true; this.linked_trials = {}; @@ -34,7 +64,7 @@ export function init() { id: t.trial_id, name: t.trial_name, bg: trial_colors[index], - fg: trial_colors_text[index] + fg: trial_colors_text[index], }; }); } @@ -43,33 +73,47 @@ export function init() { return this.linked_trials; } + get_pheno_colors () { + return colors; + } + format_brapi_post_object() { let brapi_post_plots = []; let count = 1; - for (let plot of this.plot_arr.filter(plot => plot.type == "filler")) { + for (let plot of this.plot_arr.filter((plot) => plot.type == "filler")) { brapi_post_plots.push({ - "additionalInfo": { - "invert_row_checkmark": document.getElementById("invert_row_checkmark").checked, - "top_border_selection": this.meta_data.top_border_selection || false, - "left_border_selection": this.meta_data.left_border_selection || false, - "right_border_selection": this.meta_data.right_border_selection || false, - "bottom_border_selection": this.meta_data.bottom_border_selection || false, - "plot_layout": this.meta_data.plot_layout || "serpentine", + additionalInfo: { + invert_row_checkmark: document.getElementById( + "invert_row_checkmark" + ).checked, + top_border_selection: this.meta_data.top_border_selection || false, + left_border_selection: + this.meta_data.left_border_selection || false, + right_border_selection: + this.meta_data.right_border_selection || false, + bottom_border_selection: + this.meta_data.bottom_border_selection || false, + plot_layout: this.meta_data.plot_layout || "serpentine", }, - "germplasmDbId": this.meta_data.filler_accession_id, - "germplasmName": this.meta_data.filler_accession_name, - "observationUnitName": this.trial_id + " filler " + (parseInt(this.meta_data.max_level_code) + count), - "observationUnitPosition": { - "observationLevel": { - "levelCode": parseInt(this.meta_data.max_level_code) + count, - "levelName": "plot", - "levelOrder": 2 + germplasmDbId: this.meta_data.filler_accession_id, + germplasmName: this.meta_data.filler_accession_name, + observationUnitName: + this.trial_id + + " filler " + + (parseInt(this.meta_data.max_level_code) + count), + observationUnitPosition: { + observationLevel: { + levelCode: parseInt(this.meta_data.max_level_code) + count, + levelName: "plot", + levelOrder: 2, }, - "positionCoordinateX": plot.observationUnitPosition.positionCoordinateX, - "positionCoordinateY": plot.observationUnitPosition.positionCoordinateY, + positionCoordinateX: + plot.observationUnitPosition.positionCoordinateX, + positionCoordinateY: + plot.observationUnitPosition.positionCoordinateY, }, - "trialDbId": this.trial_id, - "studyDbId": this.trial_id, + trialDbId: this.trial_id, + studyDbId: this.trial_id, }); count++; } @@ -78,30 +122,38 @@ export function init() { format_brapi_put_object() { let brapi_plots = {}; - for (let plot of this.plot_arr.filter(plot => plot.type == "data")) { + for (let plot of this.plot_arr.filter((plot) => plot.type == "data")) { brapi_plots[plot.observationUnitDbId] = { - "additionalInfo": { - "invert_row_checkmark": document.getElementById("invert_row_checkmark").checked, - "top_border_selection": this.meta_data.top_border_selection || false, - "left_border_selection": this.meta_data.left_border_selection || false, - "right_border_selection": this.meta_data.right_border_selection || false, - "bottom_border_selection": this.meta_data.bottom_border_selection || false, - "plot_layout": this.meta_data.plot_layout || "serpentine", + additionalInfo: { + invert_row_checkmark: document.getElementById( + "invert_row_checkmark" + ).checked, + top_border_selection: this.meta_data.top_border_selection || false, + left_border_selection: + this.meta_data.left_border_selection || false, + right_border_selection: + this.meta_data.right_border_selection || false, + bottom_border_selection: + this.meta_data.bottom_border_selection || false, + plot_layout: this.meta_data.plot_layout || "serpentine", }, - "germplasmDbId": plot.germplasmDbId, - "germplasmName": plot.gerplasmName, - "observationUnitName": plot.observationUnitName, - "observationUnitPosition": { - "observationLevel": { - "levelCode": plot.observationUnitPosition.observationLevel.levelCode, - "levelName": "plot", - "levelOrder": 2 + germplasmDbId: plot.germplasmDbId, + germplasmName: plot.gerplasmName, + observationUnitName: plot.observationUnitName, + observationUnitPosition: { + observationLevel: { + levelCode: + plot.observationUnitPosition.observationLevel.levelCode, + levelName: "plot", + levelOrder: 2, }, - "positionCoordinateX": plot.observationUnitPosition.positionCoordinateX, - "positionCoordinateY": plot.observationUnitPosition.positionCoordinateY, + positionCoordinateX: + plot.observationUnitPosition.positionCoordinateX, + positionCoordinateY: + plot.observationUnitPosition.positionCoordinateY, }, - "trialDbId": this.trial_id, - } + trialDbId: this.trial_id, + }; } return brapi_plots; } @@ -112,10 +164,24 @@ export function init() { for (let plot of data) { plot.type = "data"; if (isNaN(parseInt(plot.observationUnitPosition.positionCoordinateY))) { - plot.observationUnitPosition.positionCoordinateY = isNaN(parseInt(plot.observationUnitPosition.observationLevelRelationships[1].levelCode)) ? plot.observationUnitPosition.observationLevelRelationships[0].levelCode : plot.observationUnitPosition.observationLevelRelationships[1].levelCode; - if (plot.observationUnitPosition.positionCoordinateY in pseudo_layout) { - pseudo_layout[plot.observationUnitPosition.positionCoordinateY] += 1; - plot.observationUnitPosition.positionCoordinateX = pseudo_layout[plot.observationUnitPosition.positionCoordinateY]; + plot.observationUnitPosition.positionCoordinateY = isNaN( + parseInt( + plot.observationUnitPosition.observationLevelRelationships[1] + .levelCode + ) + ) + ? plot.observationUnitPosition.observationLevelRelationships[0] + .levelCode + : plot.observationUnitPosition.observationLevelRelationships[1] + .levelCode; + if ( + plot.observationUnitPosition.positionCoordinateY in pseudo_layout + ) { + pseudo_layout[ + plot.observationUnitPosition.positionCoordinateY + ] += 1; + plot.observationUnitPosition.positionCoordinateX = + pseudo_layout[plot.observationUnitPosition.positionCoordinateY]; } else { pseudo_layout[plot.observationUnitPosition.positionCoordinateY] = 1; plot.observationUnitPosition.positionCoordinateX = 1; @@ -123,15 +189,19 @@ export function init() { } var obs_level = plot.observationUnitPosition.observationLevel; if (obs_level.levelName == "plot") { - plot.observationUnitPosition.positionCoordinateX = parseInt(plot.observationUnitPosition.positionCoordinateX); - plot.observationUnitPosition.positionCoordinateY = parseInt(plot.observationUnitPosition.positionCoordinateY); + plot.observationUnitPosition.positionCoordinateX = parseInt( + plot.observationUnitPosition.positionCoordinateX + ); + plot.observationUnitPosition.positionCoordinateY = parseInt( + plot.observationUnitPosition.positionCoordinateY + ); // if (plot.additionalInfo && plot.additionalInfo.type == "filler") { // plot.type = "filler"; // } else { // plot.type = "data"; // } plot_object[plot.observationUnitDbId] = plot; - } + } } this.plot_object = plot_object; } @@ -141,103 +211,56 @@ export function init() { for (let observation of observations) { let trait_name = observation.observationVariableName; if (!this.heatmap_object[trait_name]) { - this.heatmap_object[trait_name] = {[observation.observationUnitDbId]: {val: observation.value, plot_name: observation.observationUnitName, id: observation.observationDbId }}; - } else { - this.heatmap_object[trait_name][observation.observationUnitDbId] = {val: observation.value, plot_name: observation.observationUnitName, id: observation.observationDbId }; - } - } - } - - traverse_map(plot_arr, planting_or_harvesting_order_layout) { - var local_this = this; - let coord_matrix = []; - var row = this.meta_data[planting_or_harvesting_order_layout].includes('row') ? "positionCoordinateY" : "positionCoordinateX"; - var col = this.meta_data[planting_or_harvesting_order_layout].includes('row') ? "positionCoordinateX" : "positionCoordinateY"; - - for (let plot of plot_arr) { - if (!coord_matrix[plot.observationUnitPosition[row]]) { - coord_matrix[plot.observationUnitPosition[row]] = []; - coord_matrix[plot.observationUnitPosition[row]][plot.observationUnitPosition[col]] = plot; + this.heatmap_object[trait_name] = { + [observation.observationUnitDbId]: { + val: observation.value, + plot_name: observation.observationUnitName, + id: observation.observationDbId, + }, + }; } else { - coord_matrix[plot.observationUnitPosition[row]][plot.observationUnitPosition[col]] = plot; - } - } - - coord_matrix = coord_matrix.filter(plot_arr => Array.isArray(plot_arr)); - if (!document.getElementById("invert_row_checkmark").checked && this.meta_data[planting_or_harvesting_order_layout].includes('row') && planting_or_harvesting_order_layout.includes('planting')) { - if ((this.meta_data.top_border_selection && !this.meta_data.bottom_border_selection) || (!this.meta_data.top_border_selection && this.meta_data.bottom_border_selection)) { - if (this.meta_data.top_border_selection) { - var top_borders = coord_matrix.shift(); - coord_matrix.push(top_borders); - } else if (this.meta_data.bottom_border_selection) { - var bottom_borders = coord_matrix.pop(); - coord_matrix.unshift(bottom_borders); - } + this.heatmap_object[trait_name][observation.observationUnitDbId] = { + val: observation.value, + plot_name: observation.observationUnitName, + id: observation.observationDbId, + }; } } - - - if (this.meta_data[planting_or_harvesting_order_layout].includes('serpentine')) { - for (let i = 0; i < coord_matrix.length; i++) { - if (i % 2 == 1) { - coord_matrix[i].reverse(); - } - } - } - - var final_arr = []; - for (let plot_arr of coord_matrix) { - plot_arr = plot_arr.filter(plot => plot !== undefined); - if (!document.getElementById("invert_row_checkmark").checked && local_this.meta_data[planting_or_harvesting_order_layout].includes('col') && planting_or_harvesting_order_layout.includes('planting')) { - if ((local_this.meta_data.top_border_selection && !local_this.meta_data.bottom_border_selection) || (!local_this.meta_data.top_border_selection && local_this.meta_data.bottom_border_selection)) { - if (local_this.meta_data.top_border_selection) { - var top_border_plot = plot_arr.shift(); - plot_arr.push(top_border_plot); - } else if (local_this.meta_data.bottom_border_selection) { - var bottom_border_plot = plot_arr.pop(); - plot_arr.unshift(bottom_border_plot); - } - } - } - final_arr.push(...plot_arr); - } - - var csv = [ - planting_or_harvesting_order_layout == "planting_order_layout" ? 'planting_order': "harvesting_order", - 'location_name', - 'trial_name', - 'plot_number', - 'plot_name', - 'accession_name', - 'seedlot_name', - ].join(','); - csv += "\n"; - final_arr = final_arr.filter(plot => plot !== undefined); - let order_number = 1; - final_arr.forEach(function(plot) { - csv += [ - order_number++, - "\"" + plot.locationName + "\"", - plot.studyName, - plot.observationUnitPosition.observationLevel ? plot.observationUnitPosition.observationLevel.levelCode : "N/A", - plot.observationUnitName, - plot.germplasmName, - plot.seedLotName ? plot.seedLotName : '' - ].join(','); - csv += "\n"; - }); - - var hiddenElement = document.createElement('a'); - hiddenElement.href = 'data:text/csv;charset=utf-8,' + encodeURI(csv); - hiddenElement.target = '_blank'; - hiddenElement.download = `Trial_${this.trial_id}_${this.meta_data[planting_or_harvesting_order_layout]}_${planting_or_harvesting_order_layout}.csv`; - hiddenElement.click(); } - get_plot_order(type, order, include_borders) { - let k = type === 'planting' ? 'planting_order_layout' : 'harvesting_order_layout'; - this.meta_data[k] = order; - this.traverse_map(this.plot_arr.filter(plot => include_borders || plot.type !== "border"), k); + get_plot_order({ + type, + order, + start, + include_borders, + include_gaps, + include_subplots, + include_plants, + additional_properties + } = {}) { + let q = new URLSearchParams({ + trial_ids: [ + this.trial_id, + ...Object.keys(this.linked_trials).map( + (e) => this.linked_trials[e].id + ), + ].join(","), + type: type, + order: order, + start: start, + top_border: !!include_borders && !!this.meta_data.top_border_selection, + right_border: + !!include_borders && !!this.meta_data.right_border_selection, + bottom_border: + !!include_borders && !!this.meta_data.bottom_border_selection, + left_border: + !!include_borders && !!this.meta_data.left_border_selection, + gaps: !!include_gaps, + subplots: !!include_subplots, + plants: !!include_plants, + ...additional_properties, + }).toString(); + window.open(`/ajax/breeders/trial_plot_order?${q}`, "_blank"); } set_meta_data() { @@ -248,11 +271,27 @@ export function init() { var max_row = 0; var max_level_code = 0; this.plot_arr.forEach((plot) => { - max_col = plot.observationUnitPosition.positionCoordinateX > max_col ? plot.observationUnitPosition.positionCoordinateX : max_col; - min_col = plot.observationUnitPosition.positionCoordinateX < min_col ? plot.observationUnitPosition.positionCoordinateX : min_col; - max_row = plot.observationUnitPosition.positionCoordinateY > max_row ? plot.observationUnitPosition.positionCoordinateY : max_row; - min_row = plot.observationUnitPosition.positionCoordinateY < min_row ? plot.observationUnitPosition.positionCoordinateY : min_row; - max_level_code = parseInt(plot.observationUnitPosition.observationLevel.levelCode) > max_level_code ? plot.observationUnitPosition.observationLevel.levelCode : max_level_code; + max_col = + plot.observationUnitPosition.positionCoordinateX > max_col + ? plot.observationUnitPosition.positionCoordinateX + : max_col; + min_col = + plot.observationUnitPosition.positionCoordinateX < min_col + ? plot.observationUnitPosition.positionCoordinateX + : min_col; + max_row = + plot.observationUnitPosition.positionCoordinateY > max_row + ? plot.observationUnitPosition.positionCoordinateY + : max_row; + min_row = + plot.observationUnitPosition.positionCoordinateY < min_row + ? plot.observationUnitPosition.positionCoordinateY + : min_row; + max_level_code = + parseInt(plot.observationUnitPosition.observationLevel.levelCode) > + max_level_code + ? plot.observationUnitPosition.observationLevel.levelCode + : max_level_code; }); this.meta_data.min_row = min_row; this.meta_data.max_row = max_row; @@ -261,7 +300,9 @@ export function init() { this.meta_data.num_rows = max_row - min_row + 1; this.meta_data.num_cols = max_col - min_col + 1; this.meta_data.max_level_code = max_level_code; - this.meta_data.display_borders = !jQuery("#include_linked_trials_checkmark").is(":checked"); + this.meta_data.display_borders = !jQuery( + "#include_linked_trials_checkmark" + ).is(":checked"); this.meta_data.overlapping_plots = {}; } @@ -270,23 +311,53 @@ export function init() { let last_coord; for (let plot of this.plot_arr) { if (last_coord === undefined) { - last_coord = [0,1]; + last_coord = [0, 1]; } if (plot === undefined) { if (last_coord[0] < this.meta_data.max_col) { - fieldmap_hole_fillers.push(this.get_plot_format(`Empty_Space_(${last_coord[0] + 1}_${last_coord[1]})`, last_coord[0] + 1, last_coord[1])); + fieldmap_hole_fillers.push( + this.get_plot_format( + `Empty_Space_(${last_coord[0] + 1}_${last_coord[1]})`, + last_coord[0] + 1, + last_coord[1] + ) + ); last_coord = [last_coord[0] + 1, last_coord[1]]; - this.plot_object['Empty Space' + String(last_coord[0]) + String(last_coord[1])] = this.get_plot_format('empty_space', last_coord[0] + 1, last_coord[1]); + this.plot_object[ + "Empty Space" + String(last_coord[0]) + String(last_coord[1]) + ] = this.get_plot_format( + "empty_space", + last_coord[0] + 1, + last_coord[1] + ); } else { - fieldmap_hole_fillers.push(this.get_plot_format(`Empty_Space_${this.meta_data.min_col}_${last_coord[1] + 1}`, this.meta_data.min_col, last_coord[1] + 1)); + fieldmap_hole_fillers.push( + this.get_plot_format( + `Empty_Space_${this.meta_data.min_col}_${last_coord[1] + 1}`, + this.meta_data.min_col, + last_coord[1] + 1 + ) + ); last_coord = [this.meta_data.min_col, last_coord[1]]; - this.plot_object['Empty Space' + String(last_coord[0]) + String(last_coord[1])] = this.get_plot_format('empty_space', this.meta_data.min_col, last_coord[1] + 1); + this.plot_object[ + "Empty Space" + String(last_coord[0]) + String(last_coord[1]) + ] = this.get_plot_format( + "empty_space", + this.meta_data.min_col, + last_coord[1] + 1 + ); } } else { - last_coord = [plot.observationUnitPosition.positionCoordinateX, plot.observationUnitPosition.positionCoordinateY]; + last_coord = [ + plot.observationUnitPosition.positionCoordinateX, + plot.observationUnitPosition.positionCoordinateY, + ]; } } - this.plot_arr = [...this.plot_arr.filter(plot => plot !== undefined), ...fieldmap_hole_fillers]; + this.plot_arr = [ + ...this.plot_arr.filter((plot) => plot !== undefined), + ...fieldmap_hole_fillers, + ]; } check_element(selection, element_id) { @@ -294,7 +365,13 @@ export function init() { } check_elements(additionalInfo) { - var elements = ["top_border_selection", "left_border_selection", "right_border_selection", "bottom_border_selection", "invert_row_checkmark"]; + var elements = [ + "top_border_selection", + "left_border_selection", + "right_border_selection", + "bottom_border_selection", + "invert_row_checkmark", + ]; for (let element of elements) { this.check_element(additionalInfo[element], element); this.meta_data[element] = additionalInfo[element]; @@ -307,14 +384,14 @@ export function init() { let p = this.plot_arr[0]; return { type: type, - observationUnitName: this.trial_id + ' ' + type, + observationUnitName: this.trial_id + " " + type, observationUnitPosition: { positionCoordinateX: x, - positionCoordinateY: y + positionCoordinateY: y, }, locationName: p.locationName, - studyName: p.studyName - } + studyName: p.studyName, + }; } change_dimensions(cols, rows) { @@ -330,27 +407,57 @@ export function init() { if (this.meta_data.retain_layout == false) { this.meta_data.max_row = rows + this.meta_data.min_row - 1; this.meta_data.max_col = cols + this.meta_data.min_col - 1; - this.meta_data.plot_layout = this.meta_data.plot_layout ? this.meta_data.plot_layout : "serpentine"; - - this.plot_arr = this.plot_arr.filter(plot => plot.type == "data"); - this.plot_arr.sort(function(a,b) { return parseFloat(a.observationUnitPosition.observationLevel.levelCode) - parseFloat(b.observationUnitPosition.observationLevel.levelCode) }); + this.meta_data.plot_layout = this.meta_data.plot_layout + ? this.meta_data.plot_layout + : "serpentine"; + + this.plot_arr = this.plot_arr.filter((plot) => plot.type == "data"); + this.plot_arr.sort(function (a, b) { + return ( + parseFloat(a.observationUnitPosition.observationLevel.levelCode) - + parseFloat(b.observationUnitPosition.observationLevel.levelCode) + ); + }); var plot_count = 0; var row_count = 0; - for (let j = this.meta_data.min_row; j < (this.meta_data.min_row+rows); j++) { + for ( + let j = this.meta_data.min_row; + j < this.meta_data.min_row + rows; + j++ + ) { row_count++; - var swap_columns = this.meta_data.plot_layout == "serpentine" && j % 2 === 0; + var swap_columns = + this.meta_data.plot_layout == "serpentine" && j % 2 === 0; var col_count = 0; - for (let i = this.meta_data.min_col; i < (this.meta_data.min_col+cols); i++) { + for ( + let i = this.meta_data.min_col; + i < this.meta_data.min_col + cols; + i++ + ) { col_count++; var row = j; var col = swap_columns ? this.meta_data.max_col - col_count + 1 : i; - if (plot_count >= this.plot_arr.length && this.meta_data.filler_accession_id) { + if ( + plot_count >= this.plot_arr.length && + this.meta_data.filler_accession_id + ) { this.meta_data.post = true; - this.plot_arr[plot_count] = this.get_plot_format('filler', col, row); - } else if (plot_count < this.plot_arr.length && this.plot_arr[plot_count].observationUnitPosition) { - this.plot_arr[plot_count].observationUnitPosition.positionCoordinateX = col; - this.plot_arr[plot_count].observationUnitPosition.positionCoordinateY = row; + this.plot_arr[plot_count] = this.get_plot_format( + "filler", + col, + row + ); + } else if ( + plot_count < this.plot_arr.length && + this.plot_arr[plot_count].observationUnitPosition + ) { + this.plot_arr[ + plot_count + ].observationUnitPosition.positionCoordinateX = col; + this.plot_arr[ + plot_count + ].observationUnitPosition.positionCoordinateY = row; } plot_count++; } @@ -359,16 +466,35 @@ export function init() { } add_corners() { - var add_corner = (condition_1, condition_2, x,y) => { + var add_corner = (condition_1, condition_2, x, y) => { if (condition_1 && condition_2) { this.plot_arr.push(this.get_plot_format("border", x, y)); } - } - add_corner(this.meta_data.top_border_selection, this.meta_data.left_border_selection, this.meta_data.min_col - 1, this.meta_data.min_row - 1); - add_corner(this.meta_data.top_border_selection, this.meta_data.right_border_selection, this.meta_data.max_col + 1, this.meta_data.min_row - 1); - add_corner(this.meta_data.bottom_border_selection, this.meta_data.left_border_selection, this.meta_data.min_col - 1, this.meta_data.max_row + 1); - add_corner(this.meta_data.bottom_border_selection, this.meta_data.right_border_selection, this.meta_data.max_col + 1, this.meta_data.max_row + 1); - + }; + add_corner( + this.meta_data.top_border_selection, + this.meta_data.left_border_selection, + this.meta_data.min_col - 1, + this.meta_data.min_row - 1 + ); + add_corner( + this.meta_data.top_border_selection, + this.meta_data.right_border_selection, + this.meta_data.max_col + 1, + this.meta_data.min_row - 1 + ); + add_corner( + this.meta_data.bottom_border_selection, + this.meta_data.left_border_selection, + this.meta_data.min_col - 1, + this.meta_data.max_row + 1 + ); + add_corner( + this.meta_data.bottom_border_selection, + this.meta_data.right_border_selection, + this.meta_data.max_col + 1, + this.meta_data.max_row + 1 + ); } add_border(border_element, row_or_col, min_or_max) { var start_iter; @@ -383,27 +509,49 @@ export function init() { if (this.meta_data[border_element]) { for (let i = start_iter; i <= end_iter; i++) { - this.plot_arr.push(this.get_plot_format("border", row_or_col == "row" ? i : min_or_max, row_or_col == "row" ? min_or_max : i)); + this.plot_arr.push( + this.get_plot_format( + "border", + row_or_col == "row" ? i : min_or_max, + row_or_col == "row" ? min_or_max : i + ) + ); } } } add_borders() { - if ( this.meta_data.display_borders ) { - this.add_border("left_border_selection", "col", this.meta_data.min_col - 1); - this.add_border("top_border_selection", "row", this.meta_data.min_row - 1); - this.add_border("right_border_selection", "col", this.meta_data.max_col + 1); - this.add_border("bottom_border_selection", "row", this.meta_data.max_row + 1); + if (this.meta_data.display_borders) { + this.add_border( + "left_border_selection", + "col", + this.meta_data.min_col - 1 + ); + this.add_border( + "top_border_selection", + "row", + this.meta_data.min_row - 1 + ); + this.add_border( + "right_border_selection", + "col", + this.meta_data.max_col + 1 + ); + this.add_border( + "bottom_border_selection", + "row", + this.meta_data.max_row + 1 + ); this.add_corners(); } } - transpose() { - this.plot_arr = this.plot_arr.filter((plot) => plot.type != "border") + this.plot_arr = this.plot_arr.filter((plot) => plot.type != "border"); this.plot_arr.map((plot) => { let tempX = plot.observationUnitPosition.positionCoordinateX; - plot.observationUnitPosition.positionCoordinateX = plot.observationUnitPosition.positionCoordinateY; + plot.observationUnitPosition.positionCoordinateX = + plot.observationUnitPosition.positionCoordinateY; plot.observationUnitPosition.positionCoordinateY = tempX; }); @@ -418,15 +566,14 @@ export function init() { let tempNumCols = this.meta_data.num_cols; this.meta_data.num_cols = this.meta_data.num_rows; this.meta_data.num_rows = tempNumCols; - d3.select("svg").remove(); this.add_borders(); this.render(); - } clickcancel() { - var event = d3.dispatch('click', 'dblclick'); + + var event = d3.dispatch("click", "dblclick"); function cc(selection) { var down, tolerance = 5, @@ -435,41 +582,78 @@ export function init() { function dist(a, b) { return Math.sqrt(Math.pow(a[0] - b[0], 2), Math.pow(a[1] - b[1], 2)); } - selection.on('mousedown', function() { + + selection.on("mousedown", function () { down = d3.mouse(document.body); last = +new Date(); }); - selection.on('mouseup', function() { + selection.on("mouseup", function () { + if (dist(down, d3.mouse(document.body)) > tolerance) { return; } else { if (wait) { window.clearTimeout(wait); wait = null; - event.dblclick(d3.event); + event.call("dblclick", this, d3.event); } else { - wait = window.setTimeout((function(e) { - return function() { - event.click(e); - wait = null; - }; - })(d3.event), 300); + + wait = window.setTimeout( + (function (e) { + return function () { + event.call("click", this, e); + wait = null; + }; + })(d3.event), + 300 + ); } } }); - }; - return d3.rebind(cc, event, 'on'); + } + // return d3.rebind(cc, event, 'on'); + return _rebind(cc, event, "on"); + + // Copies a variable number of methods from source to target. + function _rebind(target, source) { + + var i = 1, + n = arguments.length, + method; + while (++i < n) + target[(method = arguments[i])] = d3_rebind( + target, + source, + source[method] + ); + return target; + } + + // Method is assumed to be a standard D3 getter-setter: + // If passed with no arguments, gets the value. + // If passed with arguments, sets the value and returns the target. + function d3_rebind(target, source, method) { + + return function () { + var value = method.apply(source, arguments); + return arguments.length ? target : value; + }; + } } heatmap_plot_click(plot, heatmap_object, trait_name) { if (d3.event && d3.event.detail > 1) { return; - } else if (trait_name in heatmap_object && heatmap_object[trait_name][plot.observationUnitDbId]) { + } else if ( + trait_name in heatmap_object && + heatmap_object[trait_name][plot.observationUnitDbId] + ) { let val, plot_name, pheno_id; val = heatmap_object[trait_name][plot.observationUnitDbId].val; - plot_name = heatmap_object[trait_name][plot.observationUnitDbId].plot_name; + plot_name = + heatmap_object[trait_name][plot.observationUnitDbId].plot_name; pheno_id = heatmap_object[trait_name][plot.observationUnitDbId].id; - jQuery("#suppress_plot_pheno_dialog").modal("show"); + jQuery("#suppress_plot_pheno_dialog").modal("show"); jQuery("#myplot_name").html(plot_name); jQuery("#pheno_value").html(val); jQuery("#mytrait_id").html(trait_name); @@ -481,56 +665,245 @@ export function init() { if (d3.event && d3.event.detail > 1) { return; } else { - function btnClick(n){ - if (n.length == 0){ + function btnClick(n) { + if (n.length == 0) { jQuery("#hm_view_plot_image_submit").addClass("disabled"); } else { jQuery("#hm_view_plot_image_submit").removeClass("disabled"); } - return true; + return true; } if (plot.type == "data") { + var image_ids = plot.plotImageDbIds || []; var replace_accession = plot.germplasmName; var replace_plot_id = plot.observationUnitDbId; - var replace_plot_name = plot.observationUnitName;plot - var replace_plot_number = plot.observationUnitPosition.observationLevel.levelCode; - - jQuery('#plot_image_ids').html(image_ids); - jQuery('#hm_replace_accessions_link').find('button').trigger('click'); - jQuery("#hm_replace_accessions_link").on("click", function(){ btnClick(image_ids); }); - jQuery('#hm_edit_plot_information').html('Selected Plot Information: '); - jQuery('#hm_edit_plot_name').html(replace_plot_name); - jQuery('#hm_edit_plot_number').html(replace_plot_number); - var old_plot_id = jQuery('#hm_edit_plot_id').html(replace_plot_id); - var old_plot_accession = jQuery('#hm_edit_plot_accession').html(replace_accession); - jQuery('#hm_replace_plot_accessions_dialog').modal('show'); + var replace_plot_name = plot.observationUnitName; + plot; + var replace_plot_number = + plot.observationUnitPosition.observationLevel.levelCode; + + jQuery("#plot_image_ids").html(image_ids); + jQuery("#hm_replace_accessions_link").find("button").trigger("click"); + jQuery("#hm_replace_accessions_link").on("click", function () { + btnClick(image_ids); + }); + jQuery("#hm_edit_plot_information").html( + "Selected Plot Information: " + ); + jQuery("#hm_plot_name").html(replace_plot_name); + jQuery("#hm_plot_number").html(replace_plot_number); + var old_plot_id = jQuery("#hm_plot_id").html(replace_plot_id); + var old_plot_accession = jQuery("#hm_plot_accession").html( + replace_accession + ); + + jQuery("#hm_plot_details_modal").modal("show"); + jQuery('#hm_plot_structure_container').hide(); + + new jQuery.ajax({ + url: '/stock/get_plot_contents/'+replace_plot_id, + success: function (response) { + jQuery("#working_modal").modal("hide"); + if (response.error) { + alert("Error retrieving plot contents " + response.error); + } else { + + function createCollapsibleList(obj) { + let ul = document.createElement("ul");; + + const right_arrow = "▶"; + const down_arrow = "▼"; + + for (const key of Object.keys(obj).sort()) { + if (obj.hasOwnProperty(key)) { + let li = document.createElement("li"); + + if (typeof obj[key] === "object" && obj[key] !== null) { + let arrow = document.createElement("span"); + arrow.innerHTML = right_arrow; + arrow.classList.add("arrow"); + + let span = document.createElement("span"); + span.textContent = key; + span.classList.add("collapsible"); + span.prepend(arrow); + + span.onclick = function () { + nestedUl.classList.toggle("hidden"); + arrow.innerHTML = nestedUl.classList.contains("hidden") ? right_arrow : down_arrow; + }; + + li.appendChild(span); + + let nestedUl = createCollapsibleList(obj[key]); + nestedUl.classList.add("hidden"); + li.appendChild(nestedUl); + } else { + li.textContent = `${key}: ${obj[key]}`; + } + + ul.appendChild(li); + } + } + return ul; + } + + function createTableWithSubplots(obj) { + // the idea will be to create two tables and then call createTable for each subplot. But first we need to figure out if table should be wide or tall + + let max_row = 1; + let max_col = 1; + for (let subplot in obj["has"]) { + for (let plant in obj["has"][subplot]["has"]) { + let row = obj["has"][subplot]["has"][plant]["attributes"]["row_number"]["value"]; + let col = obj["has"][subplot]["has"][plant]["attributes"]["col_number"]["value"]; + if (row > max_row) { + max_row = row; + } + if (col > max_col) { + max_col = col; + } + } + } + + let subplot_map = [""]; + + for (let subplot of Object.keys(obj["has"]).sort()) { + subplot_map.push(""); + } + + subplot_map.push(""); + + return(subplot_map.join("")); + } + + function createTable(obj) { + let max_row = 1; + let max_col = 1; + let coord_dictionary = []; + for (let plant in obj["has"]) { + let row = obj["has"][plant]["attributes"]["row_number"]["value"]; + let col = obj["has"][plant]["attributes"]["col_number"]["value"]; + if (row > max_row) { + max_row = row; + } + if (col > max_col) { + max_col = col; + } + coord_dictionary[""+row+","+col+""] = plant; + } + + // table will have max_row + 1 rows and max_col + 1 cols + + let table_elems = ["
    " + subplot + "
    "); + subplot_map.push(createTable(obj["has"][subplot])); + subplot_map.push("
    "]; + + for (let row = max_row; row >= 0; row--){ + table_elems.push(""); + for (let col = 0; col <= max_col; col++){ + if (row == 0) { // add headers + if (col == 0) {// bottom left should be empty + table_elems.push(""); + } else { + table_elems.push(""); + } + } else { // Not last row, not a header row + if (col == 0) {// row header + table_elems.push(""); + } else {// normal plant + let coord = "" + row + "," + col + ""; + if (coord in coord_dictionary) { + table_elems.push(""); + } else { + table_elems.push(""); + } + } + } + } + table_elems.push(""); + } + + table_elems.push("
    "+col+""+row+""+coord_dictionary["" + row + "," + col + ""]+"empty space
    "); + + return(table_elems.join("")); + } + + let plot_structure = JSON.parse(response.data); + console.dir(plot_structure); + // Check if there are plants with row and column data. If yes, then we do a map display. If not, then we do a list display + let display_layout = false; + let structure; + for (let key in plot_structure["has"]) { + if (plot_structure["has"][key]["type"] == "subplot") { + for (let subkey in plot_structure["has"][key]["has"]) { + if (plot_structure["has"][key]["has"][subkey]["type"] == "plant") { + structure = "plot:subplot:plant"; + if (plot_structure["has"][key]["has"][subkey]["attributes"]?.["row_number"]["value"] > 0) { + display_layout = true; + } + } + } + } else if (plot_structure["has"][key]["type"] == "plant") { + structure = "plot:plant"; + if (plot_structure["has"][key]["attributes"]?.["row_number"]["value"] > 0) { + display_layout = true; + } + } else { + structure = "plot" + } + } + + let hm_plot_structure_data_container = document.getElementById("hm_plot_structure_data_container"); + + delete plot_structure["id"]; + delete plot_structure["type"]; + delete plot_structure["name"]; + + if (display_layout) { + if (structure == "plot:subplot:plant") { + hm_plot_structure_data_container.innerHTML = createTableWithSubplots(plot_structure); + } else if (structure == "plot:plant") { + hm_plot_structure_data_container.innerHTML = createTable(plot_structure); + } + hm_plot_structure_data_container.prepend(createCollapsibleList(plot_structure)); + } else { + hm_plot_structure_data_container.replaceChildren(createCollapsibleList(plot_structure)); + } + jQuery('#hm_plot_structure_container').show(); + } + }, + error: function (error) { + jQuery("#working_modal").modal("hide"); + alert("Error retrieving plot contents: " + error); + }, + }); new jQuery.ajax({ - type: 'POST', - url: '/ajax/breeders/trial/'+ trial_id +'/retrieve_plot_images', + type: "POST", + url: "/ajax/breeders/trial/" + trial_id + "/retrieve_plot_images", dataType: "json", data: { - 'image_ids': JSON.stringify(image_ids), - 'plot_name': replace_plot_name, - 'plot_id': replace_plot_id, + image_ids: JSON.stringify(image_ids), + plot_name: replace_plot_name, + plot_id: replace_plot_id, }, success: function (response) { - jQuery('#working_modal').modal("hide"); - var images = response.image_html; - if (response.error) { - alert("Error Retrieving Plot Images: "+response.error); - } - else { - jQuery("#show_plot_image_ids").html(images); - - // jQuery('#view_plot_image_dialog').modal("show"); - } + jQuery("#working_modal").modal("hide"); + var images = response.image_html; + if (response.error) { + alert("Error Retrieving Plot Images: " + response.error); + } else { + jQuery("#show_plot_image_ids").html(images); + + // jQuery('#view_plot_image_dialog').modal("show"); + } }, error: function () { - jQuery('#working_modal').modal("hide"); - alert('An error occurred retrieving plot images'); - } + jQuery("#working_modal").modal("hide"); + alert("An error occurred retrieving plot images"); + }, }); } } @@ -539,51 +912,97 @@ export function init() { addEventListeners() { let LocalThis = this; let transposeBtn = document.getElementById("transpose_fieldmap"); - transposeBtn.onclick = function() { + transposeBtn.onclick = function () { LocalThis.transpose(); - } + }; } - + FieldMap() { this.addEventListeners(); var cc = this.clickcancel(); - const colors = ["#ffffd9","#edf8b1","#c7e9b4","#7fcdbb","#41b6c4","#1d91c0","#225ea8","#253494","#081d58"]; + // this is where colors was originally + var trait_name = this.heatmap_selection; var heatmap_object = this.heatmap_object; - var plot_click = !this.heatmap_selected ? this.fieldmap_plot_click : this.heatmap_plot_click; + var plot_click = !this.heatmap_selected + ? this.fieldmap_plot_click + : this.heatmap_plot_click; var trait_vals = []; var local_this = this; if (this.heatmap_selected) { - let plots_with_selected_trait = heatmap_object[trait_name]; + let plots_with_selected_trait = heatmap_object[trait_name] || {}; + var has_negatives = 0; + var has_positives = 0; for (let obs_unit of Object.values(plots_with_selected_trait)) { - trait_vals.push(obs_unit.val); + trait_vals.push(Number(obs_unit.val)); + if (obs_unit.val < 0) { + has_negatives = 1; + } + if (obs_unit.val > 0) { + has_positives = 1; + } + } + + var domain; + var power = 1; + + function pearsonSkewness(arr) { + arr.sort((a, b) => a - b); + const median = arr[Math.floor(arr.length / 2)]; + const mean = arr.reduce((sum, val) => sum + val, 0) / arr.length; + const stdDev = Math.sqrt(arr.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / arr.length); + + return (3 * (mean - median)) / stdDev; } - var colorScale = d3.scale.quantile() - .domain(trait_vals) - .range(colors); + + var skew = pearsonSkewness(trait_vals); + console.log("Skew: " + skew); + if (skew > 0.5) { + power = 0.5 + } + + if (has_negatives == 1 && has_positives == 0) { + colors = ['darkblue', 'white']; + domain = [Math.min(...trait_vals), 0]; + } else if (has_negatives == 0 && has_positives == 1) { + colors = ['white', 'darkred']; + domain = [0, Math.max(...trait_vals)]; + } else { + colors = ['darkblue', 'white', 'darkred']; + domain = [Math.min(...trait_vals), 0, Math.max(...trait_vals)]; + } + colorScale = d3.scalePow().exponent(power).domain(domain).range(colors).interpolate(d3.interpolateRgb); } - var is_plot_overlapping = function(plot) { - if ( plot.observationUnitPosition ) { + var is_plot_overlapping = function (plot) { + if (plot.observationUnitPosition) { let k = `${plot.observationUnitPosition.positionCoordinateX}-${plot.observationUnitPosition.positionCoordinateY}`; - return Object.keys(local_this.meta_data.overlapping_plots).includes(k); + return Object.keys(local_this.meta_data.overlapping_plots).includes( + k + ); } return false; - } + }; - var get_fieldmap_plot_color = function(plot) { + var get_fieldmap_plot_color = function (plot) { var color; if (plot.observationUnitPosition.observationLevelRelationships) { - if ( is_plot_overlapping(plot) ) { + if (is_plot_overlapping(plot)) { color = "#000"; - } - else if (plot.observationUnitPosition.entryType == "check") { + } else if (plot.observationUnitPosition.entryType == "check") { color = "#6a5acd"; - } else if (plot.observationUnitPosition.observationLevelRelationships[1].levelCode % 2 == 0) { + } else if ( + plot.observationUnitPosition.observationLevelRelationships[1] + .levelCode % + 2 == + 0 + ) { color = "#c7e9b4"; - } else if (plot.observationUnitName.includes(local_this.trial_id + " filler")) { - color = "lightgrey"; + } else if ( + plot.observationUnitName.includes(local_this.trial_id + " filler") + ) { + color = "lightgrey"; } else { color = "#41b6c4"; } @@ -591,25 +1010,33 @@ export function init() { color = "lightgrey"; } return color; - } - - var get_heatmap_plot_color = function(plot) { + }; + + var get_heatmap_plot_color = function (plot) { var color; - if ( is_plot_overlapping(plot) ) { + if (is_plot_overlapping(plot)) { color = "#000"; - } - else if (!plot.observationUnitPosition.observationLevel) { + } else if (!plot.observationUnitPosition.observationLevel) { color = "lightgrey"; } else { - color = heatmap_object[trait_name][plot.observationUnitDbId] ? colorScale(heatmap_object[trait_name][plot.observationUnitDbId].val) : "white"; + var cs = heatmap_object.hasOwnProperty(trait_name) && heatmap_object[trait_name].hasOwnProperty(plot.observationUnitDbId) + ? colorScale(heatmap_object[trait_name][plot.observationUnitDbId].val) + : "darkgrey"; + color = cs ? cs : "lightgrey"; } return color; - } - var get_stroke_color = function(plot) { + }; + + var get_stroke_color = function (plot) { var stroke_color; if (plot.observationUnitPosition.observationLevel) { - if (plot.observationUnitPosition.observationLevelRelationships[0].levelCode % 2 == 0) { - stroke_color = "red" + if ( + plot.observationUnitPosition.observationLevelRelationships[0] + .levelCode % + 2 == + 0 + ) { + stroke_color = "red"; } else { stroke_color = "green"; } @@ -617,110 +1044,155 @@ export function init() { stroke_color = "#666"; } return stroke_color; - } + }; - var get_plot_message = function(plot) { - let html = ''; - if ( is_plot_overlapping(plot) ) { + var get_plot_message = function (plot) { + let html = ""; + if (is_plot_overlapping(plot)) { let k = `${plot.observationUnitPosition.positionCoordinateX}-${plot.observationUnitPosition.positionCoordinateY}`; let plots = local_this.meta_data.overlapping_plots[k]; - html += `Overlapping Plots: ${plots.join(', ')}`; - } - else { - html += jQuery("#include_linked_trials_checkmark").is(":checked") ? - `Trial Name: ${plot.studyName}
    ` : - ""; + html += `Overlapping Plots: ${plots.join(", ")}`; + } else { + html += jQuery("#include_linked_trials_checkmark").is(":checked") + ? `Trial Name: ${plot.studyName}
    ` + : ""; html += `Plot Name: ${plot.observationUnitName}
    `; - if ( plot.type == "data" ) { + if (plot.type == "data") { html += `Plot Number: ${plot.observationUnitPosition.observationLevel.levelCode}
    Block Number: ${plot.observationUnitPosition.observationLevelRelationships[1].levelCode}
    - Rep Number: ${plot.observationUnitPosition.observationLevelRelationships[0].levelCode}
    - Accession Name: ${plot.germplasmName}`; + Rep Number: ${plot.observationUnitPosition.observationLevelRelationships[0].levelCode}
    `; + if (plot.germplasmName) { + html += `Accession Name: ${plot.germplasmName}`; + } else if (plot.crossName) { + html += `Cross Unique ID: ${plot.crossName}`; + } else if (plot.additionalInfo.familyName) { + html += `Family Name: ${plot.additionalInfo.familyName}`; + } + if ( local_this.heatmap_selected ) { - let v = heatmap_object[trait_name][plot.observationUnitDbId].val; - v = Math.round((parseFloat(v) + Number.EPSILON) * 100) / 100 + let v = 'NA'; + if ( heatmap_object.hasOwnProperty(trait_name) && heatmap_object[trait_name].hasOwnProperty(plot.observationUnitDbId) ) { + v = heatmap_object[trait_name][plot.observationUnitDbId].val; + v = isNaN(v) ? v : Math.round((parseFloat(v) + Number.EPSILON) * 100) / 100; + } + html += `
    Trait Name: ${local_this.heatmap_selection}`; html += `
    Trait Value: ${v}`; } } } return html; - } + }; - var handle_mouseover = function(d) { + var handle_mouseover = function (d) { if (d.observationUnitPosition.observationLevel) { d3.select(`#fieldmap-plot-${d.observationUnitDbId}`) - .style('fill', 'green') - .style('cursor', 'pointer') + + .style("fill", "green") + .style("cursor", "pointer") + .style("stroke-width", 3) - .style("stroke", '#000000'); - tooltip.style('opacity', .9) - .style("left", (window.event.clientX+25) + "px") + .style("stroke", "#000000"); + tooltip + .style("opacity", 0.9) + .style("left", window.event.clientX + 25 + "px") .style("top", window.event.clientY + "px") .html(get_plot_message(d)); } - } + }; - var handle_mouseout = function(d) { + + var handle_mouseout = function (d) { d3.select(`#fieldmap-plot-${d.observationUnitDbId}`) - .style('fill', !isHeatMap ? get_fieldmap_plot_color(d) : get_heatmap_plot_color(d)) - .style('cursor', 'default') + .style( + "fill", + !isHeatMap ? get_fieldmap_plot_color(d) : get_heatmap_plot_color(d) + ) + .style("cursor", "default") .style("stroke-width", 2) .style("stroke", get_stroke_color); - tooltip.style('opacity', 0); + tooltip.style("opacity", 0); plots.exit().remove(); - } + }; - var plot_x_coord = function(plot) { - return plot.observationUnitPosition.positionCoordinateX - min_col + col_increment + 1 - } + var plot_x_coord = function (plot) { + return ( + plot.observationUnitPosition.positionCoordinateX - + min_col + + col_increment + + 1 + ); + }; - var plot_y_coord = function(plot) { - let y = plot.observationUnitPosition.positionCoordinateY - min_row + row_increment; - if ( plot.type !== "border" && document.getElementById("invert_row_checkmark").checked !== true ) { + var plot_y_coord = function (plot) { + let y = + plot.observationUnitPosition.positionCoordinateY - + min_row + + row_increment; + if ( + plot.type !== "border" && + document.getElementById("invert_row_checkmark").checked !== true + ) { y = num_rows - y - 1; } return y; - } + }; - var width = this.meta_data.display_borders && this.meta_data.left_border_selection ? - this.meta_data.num_cols + 3 : - this.meta_data.num_cols + 2; - width = this.meta_data.display_borders && this.meta_data.right_border_selection ? - width + 1 : - width; - var height = this.meta_data.display_borders && this.meta_data.top_border_selection ? - this.meta_data.num_rows + 3 : - this.meta_data.num_rows + 2; - height = this.meta_data.display_borders && this.meta_data.bottom_border_selection ? - height + 1 : - height; - var row_increment = this.meta_data.invert_row_checkmark ? - 1 : - 0; - row_increment = this.meta_data.display_borders && this.meta_data.top_border_selection && this.meta_data.invert_row_checkmark ? - row_increment + 1 : - row_increment; - var y_offset = this.meta_data.display_borders && this.meta_data.top_border_selection && !this.meta_data.invert_row_checkmark ? - 50 : - 0; - var col_increment = this.meta_data.display_borders && this.meta_data.left_border_selection ? - 1 : - 0; + var width = + this.meta_data.display_borders && this.meta_data.left_border_selection + ? this.meta_data.num_cols + 3 + : this.meta_data.num_cols + 2; + width = + this.meta_data.display_borders && this.meta_data.right_border_selection + ? width + 1 + : width; + var height = + this.meta_data.display_borders && this.meta_data.top_border_selection + ? this.meta_data.num_rows + 3 + : this.meta_data.num_rows + 2; + height = + this.meta_data.display_borders && this.meta_data.bottom_border_selection + ? height + 1 + : height; + var row_increment = this.meta_data.invert_row_checkmark ? 1 : 0; + row_increment = + this.meta_data.display_borders && + this.meta_data.top_border_selection && + this.meta_data.invert_row_checkmark + ? row_increment + 1 + : row_increment; + var y_offset = + this.meta_data.display_borders && + this.meta_data.top_border_selection && + !this.meta_data.invert_row_checkmark + ? 50 + : 0; + var col_increment = + this.meta_data.display_borders && this.meta_data.left_border_selection + ? 1 + : 0; // Check the fieldmap for any overlapping plots (plots that share the same x/y coordinates) this.meta_data.overlapping_plots = {}; let plot_positions = {}; this.plot_arr.forEach((plot) => { - if ( plot.observationUnitPosition ) { + if (plot.observationUnitPosition) { let x = plot.observationUnitPosition.positionCoordinateX; let y = plot.observationUnitPosition.positionCoordinateY; - let p = plot.observationUnitPosition.observationLevel ? plot.observationUnitPosition.observationLevel.levelCode : ''; + let p = plot.observationUnitPosition.observationLevel + ? plot.observationUnitPosition.observationLevel.levelCode + : ""; let t = plot.studyName; - if ( x && y ) { + if (x && y) { let k = `${x}-${y}`; - if ( !plot_positions.hasOwnProperty(k) ) plot_positions[k] = []; - plot_positions[k].push(jQuery("#include_linked_trials_checkmark").is(":checked") ? `${p} (${t})` : p); - if ( plot_positions[k].length > 1 ) { + if (!plot_positions.hasOwnProperty(k)) plot_positions[k] = []; + plot_positions[k].push( + jQuery("#include_linked_trials_checkmark").is(":checked") + ? `${p} (${t})` + : p + ); + if (plot_positions[k].length > 1) { this.meta_data.overlapping_plots[k] = plot_positions[k]; } } @@ -734,12 +1206,16 @@ export function init() { var num_rows = this.meta_data.num_rows; var isHeatMap = this.heatmap_selected; - var grid = d3.select("#fieldmap_chart") + + var grid = d3 + .select("#fieldmap_chart") .append("svg") .attr("width", width * 50 + 20 + "px") - .attr("height", height * 50 + 20 + "px") + .attr("height", height * 50 + 20 + "px"); - var tooltip = d3.select("#fieldmap_chart") + + var tooltip = d3 + .select("#fieldmap_chart") .append("rect") .attr("id", "tooltip") .attr("class", "tooltip") @@ -748,77 +1224,115 @@ export function init() { var plots = grid.selectAll("plots").data(this.plot_arr); plots.append("title"); - plots.enter().append("rect") - .attr("x", (d) => { return plot_x_coord(d) * 50 }) - .attr("y", (d) => { return plot_y_coord(d) * 50 + 15 + y_offset }) + plots + .enter() + .append("rect") + .attr("x", (d) => { + return plot_x_coord(d) * 50; + }) + .attr("y", (d) => { + return plot_y_coord(d) * 50 + 15 + y_offset; + }) .attr("rx", 2) - .attr("id", (d) => { return `fieldmap-plot-${d.observationUnitDbId}` }) + .attr("id", (d) => { + return `fieldmap-plot-${d.observationUnitDbId}`; + }) .attr("class", "col bordered") .attr("width", 48) .attr("height", 48) .style("stroke-width", 2) .style("stroke", get_stroke_color) - .style("fill", !isHeatMap ? get_fieldmap_plot_color : get_heatmap_plot_color) + .style( + "fill", + !isHeatMap ? get_fieldmap_plot_color : get_heatmap_plot_color + ) .on("mouseover", handle_mouseover) .on("mouseout", handle_mouseout) .call(cc); - cc.on("click", (el) => { + + cc.on("click", (el) => { var plot = d3.select(el.srcElement).data()[0]; - plot_click(plot, heatmap_object, trait_name) + plot_click(plot, heatmap_object, trait_name); }); - cc.on("dblclick", (el) => { + cc.on("dblclick", (el) => { var me = d3.select(el.srcElement); var d = me.data()[0]; if (d.observationUnitDbId) { - window.open('/stock/'+d.observationUnitDbId+'/view'); + window.open("/stock/" + d.observationUnitDbId + "/view"); } }); // Add a colored band to the bottom of the plot box to indicate different trials - if ( jQuery("#include_linked_trials_checkmark").is(":checked") ) { - plots.enter().append("rect") - .attr("x", (d) => { return plot_x_coord(d) * 50 + 4 }) - .attr("y", (d) => { return plot_y_coord(d) * 50 + 54 + y_offset }) + if (jQuery("#include_linked_trials_checkmark").is(":checked")) { + plots + .enter() + .append("rect") + .attr("x", (d) => { + return plot_x_coord(d) * 50 + 4; + }) + .attr("y", (d) => { + return plot_y_coord(d) * 50 + 54 + y_offset; + }) .attr("rx", 2) .attr("width", 40) .attr("height", 6) - .style("fill", (d) => { return local_this.linked_trials[d.studyName].bg }) - .style("opacity", (d) => { return is_plot_overlapping(d) ? '0' : '100' }) + .style("fill", (d) => { + return local_this.linked_trials[d.studyName].bg; + }) + .style("opacity", (d) => { + return is_plot_overlapping(d) ? "0" : "100"; + }); } plots.append("text"); - plots.enter().append("text") - .attr("x", (d) => { return plot_x_coord(d) * 50 + 10 }) - .attr("y", (d) => { return plot_y_coord(d) * 50 + 50 + y_offset}) + plots + .enter() + .append("text") + .attr("x", (d) => { + return plot_x_coord(d) * 50 + 10; + }) + .attr("y", (d) => { + return plot_y_coord(d) * 50 + 50 + y_offset; + }) .text((d) => { - if (!(d.observationUnitName.includes(local_this.trial_id + " filler")) && d.type == "data" && !is_plot_overlapping(d) ) { + if ( + !d.observationUnitName.includes(local_this.trial_id + " filler") && + d.type == "data" && + !is_plot_overlapping(d) + ) { return d.observationUnitPosition.observationLevel.levelCode; } }) .on("mouseover", handle_mouseover) .on("mouseout", handle_mouseout); - var image_icon = function(d) { - var image = d.plotImageDbIds || []; + var image_icon = function (d) { + var image = d.plotImageDbIds || []; var plot_image; - if (image.length > 0){ - plot_image = "/static/css/images/plot_images.png"; - }else{ + if (image.length > 0) { + plot_image = "/static/css/images/plot_images.png"; + } else { plot_image = ""; } return plot_image; - } + }; - plots.enter().append("image") + plots + .enter() + .append("image") .attr("xlink:href", image_icon) - .attr("x", (d) => { return plot_x_coord(d) * 50 + 5 }) - .attr("y", (d) => { return plot_y_coord(d) * 50 + 15 + y_offset }) + .attr("x", (d) => { + return plot_x_coord(d) * 50 + 5; + }) + .attr("y", (d) => { + return plot_y_coord(d) * 50 + 15 + y_offset; + }) .attr("width", 20) .attr("height", 20) .on("mouseover", handle_mouseover) - .on("mouseout", handle_mouseout) - + .on("mouseout", handle_mouseout); + plots.exit().remove(); var row_label_arr = []; @@ -833,33 +1347,49 @@ export function init() { var row_labels_col = 1; var col_labels_row = 0; if (!this.meta_data.invert_row_checkmark) { - col_labels_row = this.meta_data.display_borders && this.meta_data.bottom_border_selection ? num_rows + 1 : num_rows; + col_labels_row = + this.meta_data.display_borders && + this.meta_data.bottom_border_selection + ? num_rows + 1 + : num_rows; row_label_arr.reverse(); } - grid.selectAll(".rowLabels") + grid + .selectAll(".rowLabels") .data(row_label_arr) - .enter().append("text") - .attr("x", (row_labels_col * 50 - 25)) - .attr("y", (label, i) => { - let y = this.meta_data.invert_row_checkmark ? i+1 : i; - y = this.meta_data.display_borders && this.meta_data.top_border_selection && this.meta_data.invert_row_checkmark ? y+1 : y; + .enter() + .append("text") + .attr("x", row_labels_col * 50 - 25) + .attr("y", (label, i) => { + let y = this.meta_data.invert_row_checkmark ? i + 1 : i; + y = + this.meta_data.display_borders && + this.meta_data.top_border_selection && + this.meta_data.invert_row_checkmark + ? y + 1 + : y; return y * 50 + 45 + y_offset; }) - .text((label) => { return label }); + .text((label) => { + return label; + }); - grid.selectAll(".colLabels") + grid + .selectAll(".colLabels") .data(col_label_arr) - .enter().append("text") + .enter() + .append("text") .attr("x", (label, i) => { - let x = label-min_col+col_increment+2; + let x = label - min_col + col_increment + 2; return x * 50 - 30; }) - .attr("y", (col_labels_row * 50) + 45 + y_offset) - .text((label) => { return label }); + .attr("y", col_labels_row * 50 + 45 + y_offset) + .text((label) => { + return label; + }); } - load() { d3.select("svg").remove(); this.change_dimensions(this.meta_data.num_cols, this.meta_data.num_rows); @@ -869,8 +1399,11 @@ export function init() { render() { jQuery("#working_modal").modal("hide"); - jQuery("#fieldmap_chart").css({ "display": "inline-block" }); - jQuery("#container_fm").css({ "display": "inline-block", "overflow": "auto" }); + jQuery("#fieldmap_chart").css({ display: "inline-block" }); + jQuery("#container_fm").css({ + display: "inline-block", + overflow: "auto", + }); jQuery("#trait_heatmap").css("display", "none"); jQuery("#container_heatmap").css("display", "none"); jQuery("#trait_heatmap").css("display", "none"); @@ -880,4 +1413,4 @@ export function init() { const mapObj = new FieldMap(); return mapObj; -} \ No newline at end of file +} diff --git a/js/source/entries/job.js b/js/source/entries/job.js new file mode 100644 index 0000000000..83a285f8d7 --- /dev/null +++ b/js/source/entries/job.js @@ -0,0 +1,114 @@ +/** +* functions for displaying submitted job +* information on the user details page +* +* @author Ryan Preble +* +*/ + +export function dismiss_job(job_id) { + if(confirm("Job dismissal removes record of this submission, but does not delete any resulting analyses or other data. Are you sure you want to proceed?")) { + $.ajax({ + url: '/ajax/job/delete/'+job_id, + success: function(response) { + if (response.error) { + alert("Error dismissing job."); + window.location.reload(); + } else { + alert("Job record deleted."); + window.location.reload(); + } + }, + error: function(response) { + alert("Error dismissing job."); + window.location.reload(); + } + }); + } +} + +export function dismiss_finished_jobs(user_id) { + if (confirm("Dismiss all finished and canceled jobs?")){ + + $.ajax({ + url: '/ajax/job/delete_finished/'+user_id, + success: function(response) { + if (response.error) { + alert("Error dismissing jobs."); + window.location.reload(); + } else { + alert("Jobs cleared."); + window.location.reload(); + } + }, + error: function(response) { + alert("Error dismissing jobs."); + window.location.reload(); + } + }); + } +} + +export function cancel_job(job_id) { + if(confirm("Are you sure you want to cancel?")) { + $.ajax({ + url: '/ajax/job/cancel/'+job_id, + success: function(response) { + if (response.error) { + alert("Error canceling job."); + } else { + alert("Job canceled."); + window.location.reload(); + } + }, + error: function(response) { + alert("Error canceling job."); + } + }); + } +} + +export function keep_recent_jobs(user_id) { + if (confirm("Dismiss old jobs?")){ + var interval = jQuery('#keep_recent_jobs_select').val(); + + $.ajax({ + url: '/ajax/job/delete_older_than/'+interval+'/'+user_id, + success: function(response) { + if (response.error) { + alert("Error dismissing jobs."); + window.location.reload(); + } else { + alert("Jobs cleared."); + window.location.reload(); + } + }, + error: function(response) { + alert("Error dismissing jobs."); + window.location.reload(); + } + }); + } +} + +export function dismiss_dead_jobs(user_id) { + if (confirm("Dismiss failed and timed out jobs?")) { + $.ajax({ + url: '/ajax/job/delete_dead_jobs/'+user_id, + success: function(response) { + if (response.error) { + alert("Error dismissing jobs."); + window.location.reload(); + } else { + alert("Jobs cleared."); + window.location.reload(); + } + }, + error: function(response) { + alert("Error dismissing jobs."); + window.location.reload(); + } + }); + } +} + diff --git a/js/source/entries/mixedmodels.js b/js/source/entries/mixedmodels.js index 54482d8de3..bbf6ca7fc7 100644 --- a/js/source/entries/mixedmodels.js +++ b/js/source/entries/mixedmodels.js @@ -4,110 +4,115 @@ import '../legacy/d3/d3Min.js'; var version = '0.01'; -export function init(main_div){ - if (!(main_div instanceof HTMLElement)){ - main_div = document.getElementById(main_div.startsWith("#") ? main_div.slice(1) : main_div); +export function init(main_div) { + if (!(main_div instanceof HTMLElement)) { + main_div = document.getElementById(main_div.startsWith("#") ? main_div.slice(1) : main_div); } var dataset_id; - alert("WELCOME TO MIXED MODELS!"); - get_select_box("datasets", "mixed_model_dataset_select", {"checkbox_name":"mixed_model_dataset_select_checkbox"}); + // alert("WELCOME TO MIXED MODELS!"); + get_select_box("datasets", "mixed_model_dataset_select", { "checkbox_name": "mixed_model_dataset_select_checkbox", "analysis_type":"Mixed Models", "show_compatibility":"yes" }); jQuery('#mixed_model_analysis_prepare_button').removeClass('active').addClass('inactive'); - $(document).on('click', 'input[name=select_engine]', function(e) { - //alert('clicked select engine!'); - get_model_string(); + $(document).on('click', 'input[name=select_engine]', function (e) { + //alert('clicked select engine!'); + get_model_string(); }); - $(document).on('click', '#open_store_adjusted_blups_dialog_button', function(e) { - $('#generic_save_analysis_dialog').modal("show"); - $('#generic_save_analysis_model_properties').val(model_properties); - $('#generic_save_analysis_protocol').val( $('#model_string').html() ); - $('#generic_save_analysis_dataset_id').val( dataset_id ); - $('#generic_save_analysis_accession_names').val(accession_names); - $('#generic_save_analysis_dataset_id').val( get_dataset_id() ); - $('#generic_save_analysis_trait_names').val( traits ); - $('#generic_save_analysis_statistical_ontology_term').val('Adjusted means from BLUPs using LMER R|SGNSTAT:0000034'); - $('#generic_save_analysis_model_language').val('R'); - $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); - $('#generic_save_analysis_model_application_version').val( version ); - $('#generic_save_analysis_model_type').val('mixed_model_lmer'); - $('#generic_save_analysis_result_values').val(adjusted_blups_data); - $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); - $('#generic_save_analysis_result_summary_values').val(result_summary); - $('#generic_save_analysis_model_training_data_file').val(input_file); - $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); + $(document).on('click', '#open_store_adjusted_blups_dialog_button', function (e) { + $('#generic_save_analysis_dialog').modal("show"); + $('#generic_save_analysis_model_properties').val(model_properties); + $('#generic_save_analysis_protocol').val($('#model_string').html()); + $('#generic_save_analysis_dataset_id').val(dataset_id); + $('#generic_save_analysis_accession_names').val(accession_names); + $('#generic_save_analysis_dataset_id').val(get_dataset_id()); + $('#generic_save_analysis_trait_names').val(traits); + $('#generic_save_analysis_statistical_ontology_term').val('Adjusted means from BLUPs using LMER R|SGNSTAT:0000034'); + $('#generic_save_analysis_model_language').val('R'); + $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); + $('#generic_save_analysis_model_application_version').val(version); + $('#generic_save_analysis_model_type').val('mixed_model_lmer'); + $('#generic_save_analysis_result_values').val(adjusted_blups_data); + $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); + $('#generic_save_analysis_result_summary_values').val(result_summary); + $('#generic_save_analysis_model_training_data_file').val(input_file); + $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); }); - $(document).on('click', '#open_store_blups_dialog_button', function(e) { - $('#generic_save_analysis_dialog').modal("show"); - $('#generic_save_analysis_model_properties').val(model_properties); - $('#generic_save_analysis_protocol').val( $('#model_string').html() ); - $('#generic_save_analysis_dataset_id').val( dataset_id ); - $('#generic_save_analysis_accession_names').val(accession_names); - $('#generic_save_analysis_dataset_id').val( get_dataset_id() ); - $('#generic_save_analysis_trait_names').val( traits ); - $('#generic_save_analysis_statistical_ontology_term').val('Phenotypic BLUPs using LMER R|SGNSTAT:0000035'); - $('#generic_save_analysis_model_language').val('R'); - $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); - $('#generic_save_analysis_model_application_version').val( version ); - $('#generic_save_analysis_model_type').val('mixed_model_lmer'); - $('#generic_save_analysis_result_values').val(blups_data); - $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); - $('#generic_save_analysis_result_summary_values').val(result_summary); - $('#generic_save_analysis_model_training_data_file').val(input_file); - $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); + $(document).on('click', '#open_store_blups_dialog_button', function (e) { + $('#generic_save_analysis_dialog').modal("show"); + $('#generic_save_analysis_model_properties').val(model_properties); + $('#generic_save_analysis_protocol').val($('#model_string').html()); + $('#generic_save_analysis_dataset_id').val(dataset_id); + $('#generic_save_analysis_accession_names').val(accession_names); + $('#generic_save_analysis_dataset_id').val(get_dataset_id()); + $('#generic_save_analysis_trait_names').val(traits); + $('#generic_save_analysis_statistical_ontology_term').val('Phenotypic BLUPs using LMER R|SGNSTAT:0000035'); + $('#generic_save_analysis_model_language').val('R'); + $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); + $('#generic_save_analysis_model_application_version').val(version); + $('#generic_save_analysis_model_type').val('mixed_model_lmer'); + $('#generic_save_analysis_result_values').val(blups_data); + $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); + $('#generic_save_analysis_result_summary_values').val(result_summary); + $('#generic_save_analysis_model_training_data_file').val(input_file); + $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); }); - $(document).on('click', '#open_store_adjusted_blues_dialog_button', function(e) { - $('#generic_save_analysis_dialog').modal("show"); - $('#generic_save_analysis_model_properties').val(model_properties); - $('#generic_save_analysis_protocol').val( $('#model_string').html() ); - $('#generic_save_analysis_dataset_id').val( dataset_id ); - $('#generic_save_analysis_accession_names').val(accession_names); - $('#generic_save_analysis_dataset_id').val( get_dataset_id() ); - $('#generic_save_analysis_trait_names').val( traits ); - $('#generic_save_analysis_statistical_ontology_term').val('Adjusted means from BLUEs using LMER R|SGNSTAT:0000036'); - $('#generic_save_analysis_model_language').val('R'); - $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); - $('#generic_save_analysis_model_application_version').val( version ); - $('#generic_save_analysis_model_type').val('mixed_model_lmer'); - $('#generic_save_analysis_result_summary_values').val(result_summary); - $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); - $('#generic_save_analysis_model_training_data_file').val(input_file); - $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); + $(document).on('click', '#open_store_adjusted_blues_dialog_button', function (e) { + $('#generic_save_analysis_dialog').modal("show"); + $('#generic_save_analysis_model_properties').val(model_properties); + $('#generic_save_analysis_protocol').val($('#model_string').html()); + $('#generic_save_analysis_dataset_id').val(dataset_id); + $('#generic_save_analysis_accession_names').val(accession_names); + $('#generic_save_analysis_dataset_id').val(get_dataset_id()); + $('#generic_save_analysis_trait_names').val(traits); + $('#generic_save_analysis_statistical_ontology_term').val('Adjusted means from BLUEs using LMER R|SGNSTAT:0000036'); + $('#generic_save_analysis_model_language').val('R'); + $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); + $('#generic_save_analysis_model_application_version').val(version); + $('#generic_save_analysis_model_type').val('mixed_model_lmer'); + $('#generic_save_analysis_result_values').val(adjusted_blues_data); + $('#generic_save_analysis_result_summary_values').val(result_summary); + $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); + $('#generic_save_analysis_model_training_data_file').val(input_file); + $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); }); - $(document).on('click', '#open_store_blues_dialog_button', function(e) { - $('#generic_save_analysis_dialog').modal("show"); - $('#generic_save_analysis_model_properties').val(model_properties); - $('#generic_save_analysis_protocol').val( $('#model_string').html() ); - $('#generic_save_analysis_dataset_id').val( dataset_id ); - $('#generic_save_analysis_accession_names').val(accession_names); - $('#generic_save_analysis_dataset_id').val( get_dataset_id() ); - $('#generic_save_analysis_trait_names').val( traits ); - $('#generic_save_analysis_statistical_ontology_term').val('Phenotypic BLUEs using LMER R|SGNSTAT:0000037'); - $('#generic_save_analysis_model_language').val('R'); - $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); - $('#generic_save_analysis_model_application_version').val( version ); - $('#generic_save_analysis_model_type').val('mixed_model_lmer'); - $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); - $('#generic_save_analysis_result_summary_values').val(result_summary); - $('#generic_save_analysis_model_training_data_file').val(input_file); - $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); + + $(document).on('click', '#open_store_blues_dialog_button', function (e) { + $('#generic_save_analysis_dialog').modal("show"); + $('#generic_save_analysis_model_properties').val(model_properties); + $('#generic_save_analysis_protocol').val($('#model_string').html()); + $('#generic_save_analysis_dataset_id').val(dataset_id); + $('#generic_save_analysis_accession_names').val(accession_names); + $('#generic_save_analysis_dataset_id').val(get_dataset_id()); + $('#generic_save_analysis_trait_names').val(traits); + $('#generic_save_analysis_statistical_ontology_term').val('Phenotypic BLUEs using LMER R|SGNSTAT:0000037'); + $('#generic_save_analysis_model_language').val('R'); + $('#generic_save_analysis_model_application_name').val('Breedbase Mixed Model Tool'); + $('#generic_save_analysis_model_application_version').val(version); + $('#generic_save_analysis_model_type').val('mixed_model_lmer'); + $('#generic_save_analysis_result_values').val(blues_data); + $('#generic_save_analysis_result_values_type').val('analysis_result_values_match_accession_names'); + $('#generic_save_analysis_result_summary_values').val(result_summary); + $('#generic_save_analysis_model_training_data_file').val(input_file); + $('#generic_save_analysis_model_archived_training_data_file_type').val('mixed_model_input_data'); }); - $('#mixed_model_analysis_prepare_button').click( function() { + $('#mixed_model_analysis_prepare_button').click(function () { + + dataset_id = get_dataset_id(); + var dataset_trait_outliers = $('#dataset_trait_outliers').is(':checked') ? 1 : 0; - dataset_id=get_dataset_id(); - if (dataset_id != false) { + if (dataset_id != false) { $.ajax({ url: '/ajax/mixedmodels/prepare', - data: { 'dataset_id' : get_dataset_id() }, - success: function(r) { + data: { 'dataset_id': get_dataset_id(),'dataset_trait_outliers': dataset_trait_outliers, }, + success: function (r) { if (r.error) { alert(r.error); } @@ -115,73 +120,77 @@ export function init(main_div){ $('#dependent_variable').html(r.dependent_variable); var html = ""; - for (var n=0; n"+r.factors[n]+"
    "; + for (var n = 0; n < r.factors.length; n++) { + html += "
    " + r.factors[n] + "
    "; } $('#factors').html(html); - for (var n=0; n
    X '+collection_name+' Term '+factor_count+'
    '; + var div = '
    X ' + collection_name + ' Term ' + factor_count + '
    '; - $('#'+collection_div).append(div); + $('#' + collection_div).append(div); - $('#'+div_name).droppable( { - drop: function( event, ui ) { + $('#' + div_name).droppable({ + drop: function (event, ui) { var droppable = $(this); - var draggable = ui.draggable; - // Move draggable into droppable - var clone = draggable.clone(); - clone.draggable({ revert: "invalid", helper:"clone" }); - clone.css("z-index",2); + var draggable = ui.draggable; + // Move draggable into droppable + var clone = draggable.clone(); + clone.draggable({ revert: "invalid", helper: "clone" }); + clone.css("z-index", 2); if (!isCloned(clone)) { - setClonedTagProperties(clone); + setClonedTagProperties(clone); } - clone.appendTo(droppable); - get_model_string(); - }}); - - $(document).on("click", "span.remove", function(e) { - this.parentNode.parentNode.remove(); get_model_string(); - }); - - - } - - function isCloned(e) { - if (e.text().includes('X')) { - return true; - } - - return false; - } -//onclick="this.parentNode.parentNode.removeChild(this.parentNode); return false;"> - function setClonedTagProperties(e) { - e.id = e.html()+'C'; - var html = 'X '+e.html(); - e.html(html); - $(document).on("click", "span.remove_factor", function(e) { this.parentNode.remove(); get_model_string()}); - } - - $('#dependent_variable').click('#dependent_variable_select', function() { - var tempfile = $('#tempfile').html(); - var trait_selected = []; - $('.trait_box:checked').each(function() { - trait_selected.push($(this).val()); + clone.appendTo(droppable); + get_model_string(); + } + }); + + $(document).on("click", "span.remove", function (e) { + this.parentNode.parentNode.remove(); get_model_string(); + }); + + + } + + function isCloned(e) { + if (e.text().includes('X')) { + return true; + } + + return false; + } + //onclick="this.parentNode.parentNode.removeChild(this.parentNode); return false;"> + function setClonedTagProperties(e) { + e.id = e.html() + 'C'; + var html = 'X ' + e.html(); + e.html(html); + $(document).on("click", "span.remove_factor", function (e) { this.parentNode.remove(); get_model_string() }); + } + + $('#dependent_variable').click('#dependent_variable_select', function () { + var tempfile = $('#tempfile').html(); + var trait_selected = []; + $('.trait_box:checked').each(function () { + trait_selected.push($(this).val()); + }); + + if (trait_selected.length > 1 || trait_selected.length == 0) { + jQuery('#trait_histogram').html('Please select only one trait at a time to see the histogram!'); + } else { + + var trait = trait_selected[0]; + + + $.ajax({ + url: '/ajax/mixedmodels/grabdata', + data: { 'file': tempfile }, + success: function (r) { + var v = { + "$schema": "https://vega.github.io/schema/vega-lite/v2.json", + "width": 200, + "height": 100, + "padding": 5, + "data": { 'values': r.data }, + "mark": "bar", + "encoding": { + "x": { + "bin": true, + "field": trait, + "type": "quantitative" + }, + "y": { + "aggregate": "count", + "type": "quantitative" + } + } + }; + + vegaEmbed("#trait_histogram", v); + }, + + + error: function (e) { alert('error!'); } + }); + + } + }); - if(trait_selected.length > 1 || trait_selected.length == 0){ - jQuery('#trait_histogram').html('Please select only one trait at a time to see the histogram!'); - } else { - - var trait = trait_selected[0]; - - - $.ajax( { - url: '/ajax/mixedmodels/grabdata', - data: { 'file' : tempfile }, - success: function(r) { - var v = { - "$schema": "https://vega.github.io/schema/vega-lite/v2.json", - "width": 200, - "height": 100, - "padding": 5, - "data": { 'values': r.data }, - "mark": "bar", - "encoding": { - "x": { - "bin": true, - "field": trait, - "type": "quantitative" - }, - "y": { - "aggregate": "count", - "type": "quantitative" - } - } - }; - - vegaEmbed("#trait_histogram", v); - }, - - - error: function(e) { alert('error!'); } - }); - - } - - }); - - $('#run_mixed_model_button').click( function() { + $('#run_mixed_model_button').click(function () { var model = $('#model_string').text(); - var fixed_factors = parse_simple_factors("fixed_factors"); - //alert("FIXED FACTORS: "+fixed_factors); - var random_factors = parse_simple_factors("random_factors"); - var engine = $('input[name="select_engine"]:checked').val(); - //alert("Engine is "+engine); - var tempfile = $('#tempfile').text(); + var fixed_factors = parse_simple_factors("fixed_factors"); + //alert("FIXED FACTORS: "+fixed_factors); + var random_factors = parse_simple_factors("random_factors"); + var engine = $('input[name="select_engine"]:checked').val(); + //alert("Engine is "+engine); + var tempfile = $('#tempfile').text(); - var dependent_variables = []; + var dependent_variables = []; - $('input[name=dependent_variable_select]:checked').each(function(){ + $('input[name=dependent_variable_select]:checked').each(function () { dependent_variables.push(jQuery(this).val()); - }); - console.log(dependent_variables); - $('#working_modal').modal("show"); - $.ajax( { + }); + console.log(dependent_variables); + $('#working_modal').modal("show"); + $.ajax({ "url": '/ajax/mixedmodels/run', - "method": "POST", + "method": "POST", "data": { "model" : model, "tempfile" : tempfile, @@ -361,43 +371,51 @@ export function init(main_div){ accession_names = JSON.stringify(r.accession_names); + adjusted_blups_data = JSON.stringify(r.adjusted_blups_data); adjusted_blues_data = JSON.stringify(r.adjusted_blues_data); + + console.log('Adjusted means:'+adjusted_blups_data); + console.log('Adjusted means:'+adjusted_blues_data); + blups_data = JSON.stringify(r.blups_data); blues_data = JSON.stringify(r.blues_data); traits = JSON.stringify(r.traits); + console.log("Traits: "+traits); input_file = r.input_file; result_summary = '{ "method" : "Breedbase mixed model analysis tool" }'; var model_properties_data = { "properties" : { "traits" : traits } } ; + console.log("traits: "+traits); model_properties = JSON.stringify(model_properties_data); + //alert("Model properties: "+model_properties); } }, - "error": function(r) { - alert(r); + "error": function (r) { + alert(r); } - }); + }); }); } function get_dataset_id() { var selected_datasets = []; - jQuery('input[name="mixed_model_dataset_select_checkbox"]:checked').each(function() { + jQuery('input[name="mixed_model_dataset_select_checkbox"]:checked').each(function () { selected_datasets.push(jQuery(this).val()); }); - if (selected_datasets.length < 1){ + if (selected_datasets.length < 1) { alert('Please select at least one dataset!'); return false; - } else if (selected_datasets.length > 1){ + } else if (selected_datasets.length > 1) { alert('Please select only one dataset!'); return false; } else { - var dataset_id=selected_datasets[0]; - return dataset_id; + var dataset_id = selected_datasets[0]; + return dataset_id; } } @@ -415,7 +433,7 @@ function extract_model_parameters() { //alert("ENGINE IS NOW: "+engine); - // var random_factors = $('#random_factors').text(); + // var random_factors = $('#random_factors').text(); // random_factors = random_factors.replace(/X /g, '","'); // random_factors = random_factors.replace(/\s/g, ''); // random_factors = random_factors.substr(3); @@ -428,17 +446,17 @@ function extract_model_parameters() { // } var dependent_variables = []; - $('input[name=dependent_variable_select]:checked').each(function(){ - dependent_variables.push(jQuery(this).val()); + $('input[name=dependent_variable_select]:checked').each(function () { + dependent_variables.push(jQuery(this).val()); }); - var json = { - 'fixed_factors' : fixed_factors, - 'fixed_factors_interaction' : interaction_factors, - 'variable_slope_intersects' : variable_slope_intersects, - 'random_factors' : random_factors, - 'dependent_variables' : dependent_variables, - 'engine' : engine + var json = { + 'fixed_factors': fixed_factors, + 'fixed_factors_interaction': interaction_factors, + 'variable_slope_intersects': variable_slope_intersects, + 'random_factors': random_factors, + 'dependent_variables': dependent_variables, + 'engine': engine }; console.log(json); return json; @@ -446,15 +464,15 @@ function extract_model_parameters() { function parse_simple_factors(simple_div) { - var factors = $('#'+simple_div).children(); + var factors = $('#' + simple_div).children(); var factor_list = new Array(); - for(var n=0; n !traitSelected.includes(trait)); + + filteredTraits.forEach(trait => { + const $checkbox = $('', { + type: 'checkbox', + name: 'new_trait_options', + value: trait + }); + + const $label = $('