From 900c155aa6c7df026e2086cb17a779fe790ace2d Mon Sep 17 00:00:00 2001 From: ghi820 Date: Tue, 7 May 2024 01:15:36 +0200 Subject: [PATCH 1/4] Cleanup media selection query and add stories --- OFMetadataToStash.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OFMetadataToStash.ps1 b/OFMetadataToStash.ps1 index 5e3a8d4..401f518 100644 --- a/OFMetadataToStash.ps1 +++ b/OFMetadataToStash.ps1 @@ -799,7 +799,7 @@ function Add-MetadataUsingOFDB{ if (!(DatabaseHasAlreadyBeenImported)){ #Select all the media (except audio) and the text the performer associated to them, if available from the OFDB - $Query = "SELECT messages.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN messages ON messages.post_id=medias.post_id UNION SELECT posts.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN posts ON posts.post_id=medias.post_id WHERE medias.media_type <> 'Audios'" + $Query = "SELECT descr.text, medias.directory, medias.filename, medias.size, medias.created_at, medias.post_id, medias.media_type FROM medias INNER JOIN (SELECT messages.text, messages.post_id FROM messages UNION SELECT posts.text, posts.post_id FROM posts UNION SELECT stories.text, stories.post_id FROM stories) as descr ON descr.post_id=medias.post_id WHERE medias.size IS NOT NULL ORDER BY medias.post_id ASC" $OF_DBpath = $currentdatabase.fullname $OFDBQueryResult = Invoke-SqliteQuery -Query $Query -DataSource $OF_DBpath From 0b3dd5f96bcab3f50b6b696d309a62a0e9fae84c Mon Sep 17 00:00:00 2001 From: ghi820 Date: Tue, 7 May 2024 01:23:11 +0200 Subject: [PATCH 2/4] Fix UPDATE for history (wrongfully referenced import_date instead of history) --- OFMetadataToStash.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OFMetadataToStash.ps1 b/OFMetadataToStash.ps1 index 401f518..05df8dd 100644 --- a/OFMetadataToStash.ps1 +++ b/OFMetadataToStash.ps1 @@ -286,7 +286,7 @@ function DatabaseHasAlreadyBeenImported{ if([datetime]$metadataLastWriteTime -gt [datetime]$performerFromHistory.import_date){ $currenttimestamp = get-date -format o try { - $historyQuery = 'UPDATE import_date SET import_date = "'+$currenttimestamp+'" WHERE history.performer = "'+$performername+'"' + $historyQuery = 'UPDATE history SET import_date = "'+$currenttimestamp+'" WHERE history.performer = "'+$performername+'"' Invoke-SqliteQuery -Query $historyQuery -DataSource $PathToHistoryFile } catch{ From 6e3679779c71c2be656be6ba8423c246a763d545 Mon Sep 17 00:00:00 2001 From: ghi820 Date: Tue, 7 May 2024 01:40:57 +0200 Subject: [PATCH 3/4] consolidate common sql mutation queries --- OFMetadataToStash.ps1 | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/OFMetadataToStash.ps1 b/OFMetadataToStash.ps1 index 05df8dd..8e03870 100644 --- a/OFMetadataToStash.ps1 +++ b/OFMetadataToStash.ps1 @@ -853,12 +853,15 @@ function Add-MetadataUsingOFDB{ if (($mediaToProcessSelector -eq 3) -and ($mediatype -eq "video")){ continue #Skip to the next item in this foreach, user only wants to process images } + + $VideoMutationBaseQuery='SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id' + $ImageMutationBaseQuery='SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id' #Depending on user preference, we want to be more/less specific with our SQL queries to the Stash DB here, as determined by this condition tree (defined in order of percieved popularity) #Normal specificity, search for videos based on having the performer name somewhere in the path and a matching filesize if ($mediatype -eq "video" -and $searchspecificity -match "normal"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { rows } }' @@ -866,7 +869,7 @@ function Add-MetadataUsingOFDB{ #Normal specificity, search for images based on having the performer name somewhere in the path and a matching filesize elseif ($mediatype -eq "image" -and $searchspecificity -match "normal"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE path LIKE ''%'+$performername+'%'' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND size = '''+$OFDBfilesize+'''") { rows } }' @@ -874,7 +877,7 @@ function Add-MetadataUsingOFDB{ #Low specificity, search for videos based on filesize only elseif ($mediatype -eq "video" -and $searchspecificity -match "low"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE size = '''+$OFDBfilesize+'''") { rows } }' @@ -882,7 +885,7 @@ function Add-MetadataUsingOFDB{ #Low specificity, search for images based on filesize only elseif ($mediatype -eq "image" -and $searchspecificity -match "low"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE size = '''+$OFDBfilesize+'''") { rows } }' @@ -891,7 +894,7 @@ function Add-MetadataUsingOFDB{ #High specificity, search for videos based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. elseif ($mediatype -eq "video" -and $searchspecificity -match "high"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { rows } }' @@ -900,7 +903,7 @@ function Add-MetadataUsingOFDB{ #High specificity, search for images based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. else{ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { rows } }' @@ -966,7 +969,7 @@ function Add-MetadataUsingOFDB{ if ($mediatype -eq "video"){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, scenes.id AS scenes_id, scenes.title AS scenes_title, scenes.details AS scenes_details FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN scenes_files ON files.id = scenes_files.file_id JOIN scenes ON scenes.id = scenes_files.scene_id path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { rows } }' @@ -976,7 +979,7 @@ function Add-MetadataUsingOFDB{ elseif ($mediatype -eq "image" ){ $StashGQL_Query = 'mutation { - querySQL(sql: "SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { + querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { rows } }' From dfaa3ef121d02e51f08fc04a538563a5dbae7b7d Mon Sep 17 00:00:00 2001 From: ghi820 Date: Tue, 7 May 2024 02:38:35 +0200 Subject: [PATCH 4/4] move some boilerplate --- OFMetadataToStash.ps1 | 117 ++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 62 deletions(-) diff --git a/OFMetadataToStash.ps1 b/OFMetadataToStash.ps1 index 8e03870..038e8f5 100644 --- a/OFMetadataToStash.ps1 +++ b/OFMetadataToStash.ps1 @@ -318,6 +318,27 @@ function DatabaseHasAlreadyBeenImported{ } } #End DatabaseHasBeenImported +function Build-GQLSQL(){ + param( + [Parameter(Mandatory=$true)][string]$query, + [Parameter(Mandatory=$false)][string]$where + ) + + if($where) { + return 'mutation { + querySQL(sql: "'+$query+' WHERE '+$where+'") { + rows + } + }' + } else { + return 'mutation { + querySQL(sql: "'+$query+'") { + rows + } + }' + } +} + #Add-MetadataUsingOFDB adds metadata to Stash using metadata databases. function Add-MetadataUsingOFDB{ #Playing it safe and asking the user to back up their database first @@ -858,57 +879,38 @@ function Add-MetadataUsingOFDB{ $ImageMutationBaseQuery='SELECT folders.path, files.basename, files.size, files.id AS files_id, folders.id AS folders_id, images.id AS images_id, images.title AS images_title FROM files JOIN folders ON files.parent_folder_id=folders.id JOIN images_files ON files.id = images_files.file_id JOIN images ON images.id = images_files.image_id' #Depending on user preference, we want to be more/less specific with our SQL queries to the Stash DB here, as determined by this condition tree (defined in order of percieved popularity) - #Normal specificity, search for videos based on having the performer name somewhere in the path and a matching filesize - if ($mediatype -eq "video" -and $searchspecificity -match "normal"){ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' - } - #Normal specificity, search for images based on having the performer name somewhere in the path and a matching filesize - elseif ($mediatype -eq "image" -and $searchspecificity -match "normal"){ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' - } - #Low specificity, search for videos based on filesize only - elseif ($mediatype -eq "video" -and $searchspecificity -match "low"){ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE size = '''+$OFDBfilesize+'''") { - rows - } - }' - } - #Low specificity, search for images based on filesize only - elseif ($mediatype -eq "image" -and $searchspecificity -match "low"){ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE size = '''+$OFDBfilesize+'''") { - rows - } - }' - } - - #High specificity, search for videos based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. - elseif ($mediatype -eq "video" -and $searchspecificity -match "high"){ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' - } - - #High specificity, search for images based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. - else{ - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' + #Normal specificity + if ($searchspecificity -match "normal") { + #Search for videos based on having the performer name somewhere in the path and a matching filesize + if ($mediatype -eq "video" -and $searchspecificity -match "normal"){ + $StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where "files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'" + } + #Search for images based on having the performer name somewhere in the path and a matching filesize + elseif ($mediatype -eq "image"){ + $StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where "path LIKE '%$performername%' AND size = '$OFDBfilesize'" + } + }elseif ($searchspecificity -match "low") { + $specwhere="size = '$OFDBfilesize'" + #Low specificity, search for videos based on filesize only + if ($mediatype -eq "video"){ + $StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere + } + #Low specificity, search for images based on filesize only + elseif ($mediatype -eq "image"){ + $StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere + } + }elseif($searchspecificity -match "high"){ + $specwhere="files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'" + #High specificity, search for videos based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. + if ($mediatype -eq "video"){ + $StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere + } + #High specificity, search for images based on matching file name between OnlyFans DB and Stash DB as well as matching the filesize. + else{ + $StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere + } } - + #Now lets try running the GQL query and see if we have a match in the Stash DB try{ $StashGQL_Result = Invoke-GraphQLQuery -Query $StashGQL_Query -Uri $StashGQL_URL -Headers $(if ($StashAPIKey){ @{ApiKey = "$StashAPIKey" }}) @@ -965,24 +967,15 @@ function Add-MetadataUsingOFDB{ #Before processing, and for the sake of accuracy, if there are multiple filesize matches (specifically for the normal specificity mode), add a filename check to the query to see if we can match more specifically. If not, just use whatever matched that initial query. if (($StashGQL_Result.data.querySQL.rows.length -gt 1) -and ($searchspecificity -match "normal") ){ + $specwhere="path LIKE '%$performername%' AND files.basename = '$OFDBfilenameForQuery' AND size = '$OFDBfilesize'" #Search for videos based on having the performer name somewhere in the path and a matching filesize (and filename in this instance) if ($mediatype -eq "video"){ - - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$VideoMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' + $StashGQL_Query = Build-GQLSQL -query $VideoMutationBaseQuery -where $specwhere } #Search for images based on having the performer name somewhere in the path and a matching filesize (and filename in this instance) elseif ($mediatype -eq "image" ){ - - $StashGQL_Query = 'mutation { - querySQL(sql: "'+$ImageMutationBaseQuery+' WHERE path LIKE ''%'+$performername+'%'' AND files.basename ='''+$OFDBfilenameForQuery+''' AND size = '''+$OFDBfilesize+'''") { - rows - } - }' + $StashGQL_Query = Build-GQLSQL -query $ImageMutationBaseQuery -where $specwhere } #Now lets try running the GQL query and try to find the file in the Stash DB