Skip to content

Improve SQLite Queries #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ghi820 wants to merge 4 commits into
base: dev-0.10
Choose a base branch
from
Open

Improve SQLite Queries #44

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
900c155
Cleanup media selection query and add stories
ghi820 May 6, 2024
0b3dd5f
Fix UPDATE for history (wrongfully referenced import_date instead of …
ghi820 May 6, 2024
6e36797
consolidate common sql mutation queries
ghi820 May 6, 2024
dfaa3ef
move some boilerplate
ghi820 May 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 60 additions & 64 deletions OFMetadataToStash.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ function DatabaseHasAlreadyBeenImported{
if([datetime]$metadataLastWriteTime -gt [datetime]$performerFromHistory.import_date){
$currenttimestamp = get-date -format o
try {
$historyQuery = 'UPDATE import_date SET import_date = "'+$currenttimestamp+'" WHERE history.performer = "'+$performername+'"'
$historyQuery = 'UPDATE history SET import_date = "'+$currenttimestamp+'" WHERE history.performer = "'+$performername+'"'
Invoke-SqliteQuery -Query $historyQuery -DataSource $PathToHistoryFile
}
catch{
Expand Down Expand Up @@ -318,6 +318,27 @@ function DatabaseHasAlreadyBeenImported{
}
} #End DatabaseHasBeenImported

function Build-GQLSQL(){
param(
[Parameter(Mandatory=$true)][string]$query,
[Parameter(Mandatory=$false)][string]$where
)

if($where) {
return 'mutation {
querySQL(sql: "'+$query+' WHERE '+$where+'") {
rows
}
}'
} else {
return 'mutation {
querySQL(sql: "'+$query+'") {
rows
}
}'
}
}

#Add-MetadataUsingOFDB adds metadata to Stash using metadata databases.
function Add-MetadataUsingOFDB{
#Playing it safe and asking the user to back up their database first
Expand Down Expand Up @@ -799,7 +820,7 @@ function Add-MetadataUsingOFDB{

if (!(DatabaseHasAlreadyBeenImported)){
#Select all the media (except audio) and the text the performer associated to them, if available from the OFDB
$Query = "SELECT messages.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN messages ON messages.post_id=medias.post_id UNION SELECT posts.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN posts ON posts.post_id=medias.post_id WHERE medias.media_type <> 'Audios'"
$Query = "SELECT descr.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN (SELECT messages.text, messages.post_id FROM messages UNION SELECT posts.text, posts.post_id FROM posts UNION SELECT stories.text, stories.post_id FROM stories) as descr ON descr.post_id=medias.post_id WHERE medias.size IS NOT NULL ORDER BY medias.post_id ASC"
$OF_DBpath = $currentdatabase.fullname
$OFDBQueryResult = Invoke-SqliteQuery -Query $Query -DataSource $OF_DBpath

Expand Down Expand Up @@ -853,59 +874,43 @@ function Add-MetadataUsingOFDB{
if (($mediaToProcessSelector -eq 3) -and ($mediatype -eq "video")){
continue #Skip to the next item in this foreach, user only wants to process images
}

$VideoMutationBaseQuery='SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id'
$ImageMutationBaseQuery='SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id'

#Depending on user preference, we want to be more/less specific with our SQL queries to the Stash DB here, as determined by this condition tree (defined in order of percieved popularity)
#Normal specificity, search for videos based on having the performer name somewhere in the path and a matching filesize
if ($mediatype -eq "video" -and $searchspecificity -match "normal"){
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
}
#Normal specificity, search for images based on having the performer name somewhere in the path and a matching filesize
elseif ($mediatype -eq "image" -and $searchspecificity -match "normal"){
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE path LIKE ''%'+$performername+'%'' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
}
#Low specificity, search for videos based on filesize only
elseif ($mediatype -eq "video" -and $searchspecificity -match "low"){
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE size = '''+$OFDBfilesize+'''") {
rows
}
}'
}
#Low specificity, search for images based on filesize only
elseif ($mediatype -eq "image" -and $searchspecificity -match "low"){
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE size = '''+$OFDBfilesize+'''") {
rows
}
}'
}

#High specificity, search for videos based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize.
elseif ($mediatype -eq "video" -and $searchspecificity -match "high"){
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
}

#High specificity, search for images based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize.
else{
$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
#Normal specificity
if ($searchspecificity -match "normal") {
#Search for videos based on having the performer name somewhere in the path and a matching filesize
if ($mediatype -eq "video" -and $searchspecificity -match "normal"){
$StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where "files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'"
}
#Search for images based on having the performer name somewhere in the path and a matching filesize
elseif ($mediatype -eq "image"){
$StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where "path LIKE '%$performername%' AND size = '$OFDBfilesize'"
}
}elseif ($searchspecificity -match "low") {
$specwhere="size = '$OFDBfilesize'"
#Low specificity, search for videos based on filesize only
if ($mediatype -eq "video"){
$StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere
}
#Low specificity, search for images based on filesize only
elseif ($mediatype -eq "image"){
$StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere
}
}elseif($searchspecificity -match "high"){
$specwhere="files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'"
#High specificity, search for videos based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize.
if ($mediatype -eq "video"){
$StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere
}
#High specificity, search for images based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize.
else{
$StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere
}
}

#Now lets try running the GQL query and see if we have a match in the Stash DB
try{
$StashGQL_Result = Invoke-GraphQLQuery -Query $StashGQL_Query -Uri $StashGQL_URL -Headers $(if ($StashAPIKey){ @{ApiKey = "$StashAPIKey" }})
Expand Down Expand Up @@ -962,24 +967,15 @@ function Add-MetadataUsingOFDB{

#Before processing, and for the sake of accuracy, if there are multiple filesize matches (specifically for the normal specificity mode), add a filename check to the query to see if we can match more specifically. If not, just use whatever matched that initial query.
if (($StashGQL_Result.data.querySQL.rows.length -gt 1) -and ($searchspecificity -match "normal") ){
$specwhere="path LIKE '%$performername%' AND files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'"
#Search for videos based on having the performer name somewhere in the path and a matching filesize (and filename in this instance)
if ($mediatype -eq "video"){

$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
$StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere
}

#Search for images based on having the performer name somewhere in the path and a matching filesize (and filename in this instance)
elseif ($mediatype -eq "image" ){

$StashGQL_Query = 'mutation {
querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") {
rows
}
}'
$StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere
}

#Now lets try running the GQL query and try to find the file in the Stash DB
Expand Down