diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md index 7464c466b6..0ac2e4e4b4 100644 --- a/distributor-node/CHANGELOG.md +++ b/distributor-node/CHANGELOG.md @@ -1,3 +1,8 @@ +### 2.2.0 + +- Updates `operator:set-metadata` CLI command to set distributor-node's operational status along with other metadata. +- Adds `leader:set-node-operational-status` CLI command to set operational status of any distributor-node by Lead. + ### 2.1.1 - Bump deps @polkadot/api v10.7.1 diff --git a/distributor-node/docs/commands/leader.md b/distributor-node/docs/commands/leader.md index 9f5f6efbea..94ddf18c39 100644 --- a/distributor-node/docs/commands/leader.md +++ b/distributor-node/docs/commands/leader.md @@ -12,6 +12,7 @@ Commands for performing Distribution Working Group leader on-chain duties (like * [`joystream-distributor leader:remove-bucket-operator`](#joystream-distributor-leaderremove-bucket-operator) * [`joystream-distributor leader:set-bucket-family-metadata`](#joystream-distributor-leaderset-bucket-family-metadata) * [`joystream-distributor leader:set-buckets-per-bag-limit`](#joystream-distributor-leaderset-buckets-per-bag-limit) +* [`joystream-distributor leader:set-node-operational-status`](#joystream-distributor-leaderset-node-operational-status) * [`joystream-distributor leader:update-bag`](#joystream-distributor-leaderupdate-bag) * [`joystream-distributor leader:update-bucket-mode`](#joystream-distributor-leaderupdate-bucket-mode) * [`joystream-distributor leader:update-bucket-status`](#joystream-distributor-leaderupdate-bucket-status) @@ -210,6 +211,33 @@ OPTIONS _See code: [src/commands/leader/set-buckets-per-bag-limit.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/set-buckets-per-bag-limit.ts)_ +## `joystream-distributor leader:set-node-operational-status` + +Set/update distribution node operational status. Requires distribution working group leader permissions. + +``` +USAGE + $ joystream-distributor leader:set-node-operational-status + +OPTIONS + -B, --bucketId=bucketId (required) Distribution bucket ID in + {familyId}:{bucketIndex} format. + + -c, --configPath=configPath [default: ./config.yml] Path to config + JSON/YAML file (relative to current working + directory) + + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator + + -w, --workerId=workerId (required) ID of the operator (distribution + group worker) + + -y, --yes Answer "yes" to any prompt, skipping any + manual confirmations +``` + +_See code: [src/commands/leader/set-node-operational-status.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/set-node-operational-status.ts)_ + ## `joystream-distributor leader:update-bag` Add/remove distribution buckets from a bag. diff --git a/distributor-node/docs/commands/operator.md b/distributor-node/docs/commands/operator.md index 907978fb87..d60235d069 100644 --- a/distributor-node/docs/commands/operator.md +++ b/distributor-node/docs/commands/operator.md @@ -39,18 +39,24 @@ USAGE $ joystream-distributor operator:set-metadata OPTIONS - -B, --bucketId=bucketId (required) Distribution bucket ID in {familyId}:{bucketIndex} format. + -B, --bucketId=bucketId (required) Distribution bucket ID in + {familyId}:{bucketIndex} format. - -c, --configPath=configPath [default: ./config.yml] Path to config JSON/YAML file (relative to current working - directory) + -c, --configPath=configPath [default: ./config.yml] Path to config + JSON/YAML file (relative to current working + directory) - -e, --endpoint=endpoint Root distribution node endpoint + -e, --endpoint=endpoint Root distribution node endpoint - -i, --input=input Path to JSON metadata file + -i, --input=input Path to JSON metadata file - -w, --workerId=workerId (required) ID of the operator (distribution group worker) + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator - -y, --yes Answer "yes" to any prompt, skipping any manual confirmations + -w, --workerId=workerId (required) ID of the operator (distribution + group worker) + + -y, --yes Answer "yes" to any prompt, skipping any + manual confirmations DESCRIPTION Requires active distribution bucket operator worker role key. diff --git a/distributor-node/package.json b/distributor-node/package.json index 4c5950930f..cb629376bf 100644 --- a/distributor-node/package.json +++ b/distributor-node/package.json @@ -1,7 +1,7 @@ { "name": "@joystream/distributor-cli", "description": "Joystream distributor node CLI", - "version": "2.1.1", + "version": "2.2.0", "author": "Joystream contributors", "bin": { "joystream-distributor": "./bin/run" @@ -34,6 +34,7 @@ "graphql": "^15.3.0", "graphql-tag": "^2.12.6", "inquirer": "^8.1.2", + "inquirer-datepicker": "2.0.2", "js-image-generator": "^1.0.3", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.21", diff --git a/distributor-node/src/@types/inquirer-datepicker/index.d.ts b/distributor-node/src/@types/inquirer-datepicker/index.d.ts new file mode 100644 index 0000000000..5f9d649d02 --- /dev/null +++ b/distributor-node/src/@types/inquirer-datepicker/index.d.ts @@ -0,0 +1 @@ +declare module 'inquirer-datepicker' diff --git a/distributor-node/src/command-base/default.ts b/distributor-node/src/command-base/default.ts index 1e4fecd04f..b4f7247f78 100644 --- a/distributor-node/src/command-base/default.ts +++ b/distributor-node/src/command-base/default.ts @@ -1,12 +1,13 @@ import Command, { flags as oclifFlags } from '@oclif/command' -import inquirer from 'inquirer' -import ExitCodes from './ExitCodes' -import { ReadonlyConfig } from '../types/config' -import { ConfigParserService } from '../services/parsers/ConfigParserService' -import { LoggingService } from '../services/logging' +import inquirer, { DistinctQuestion } from 'inquirer' +import inquirerDatepicker from 'inquirer-datepicker' import { Logger } from 'winston' +import { LoggingService } from '../services/logging' import { BagIdParserService } from '../services/parsers/BagIdParserService' import { BucketIdParserService } from '../services/parsers/BucketIdParserService' +import { ConfigParserService } from '../services/parsers/ConfigParserService' +import { ReadonlyConfig } from '../types/config' +import ExitCodes from './ExitCodes' export const flags = { ...oclifFlags, @@ -75,6 +76,7 @@ export default abstract class DefaultCommandBase extends Command { this.logging = LoggingService.withCLIConfig() this.logger = this.logging.createLogger('CLI') this.autoConfirm = !!(process.env.AUTO_CONFIRM === 'true' || parseInt(process.env.AUTO_CONFIRM || '') || yes) + inquirer.registerPrompt('datepicker', inquirerDatepicker) } public log(message: string, ...meta: unknown[]): void { @@ -98,6 +100,21 @@ export default abstract class DefaultCommandBase extends Command { } } + async datePrompt(question: DistinctQuestion): Promise { + const { result } = await inquirer.prompt([ + { + ...question, + type: 'datepicker', + name: 'result', + clearable: true, + default: new Date().toISOString(), + }, + ]) + + const date = new Date(result) + return date + } + async finally(err: unknown): Promise { if (!err) this.exit(ExitCodes.OK) if (process.env.DEBUG === 'true') { diff --git a/distributor-node/src/commands/leader/set-node-operational-status.ts b/distributor-node/src/commands/leader/set-node-operational-status.ts new file mode 100644 index 0000000000..b95bd9e319 --- /dev/null +++ b/distributor-node/src/commands/leader/set-node-operational-status.ts @@ -0,0 +1,93 @@ +import { + INodeOperationalStatus, + ISetNodeOperationalStatus, + NodeOperationalStatus, + SetNodeOperationalStatus, +} from '@joystream/metadata-protobuf' +import AccountsCommandBase from '../../command-base/accounts' +import DefaultCommandBase, { flags } from '../../command-base/default' + +export default class LeadSetNodeOperationalStatus extends AccountsCommandBase { + static description = `Set/update distribution node operational status. Requires distribution working group leader permissions.` + + static flags = { + bucketId: flags.bucketId({ + required: true, + }), + workerId: flags.integer({ + char: 'w', + description: 'ID of the operator (distribution group worker)', + required: true, + }), + operationalStatus: flags.enum>({ + char: 'o', + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: true, + description: 'Operational status of the operator', + }), + rationale: flags.string({ + char: 'r', + description: 'Rationale for setting the operational status', + }), + ...DefaultCommandBase.flags, + } + + async run(): Promise { + const { + bucketId, + workerId, + rationale, + operationalStatus: statusType, + } = this.parse(LeadSetNodeOperationalStatus).flags + const leadKey = await this.getDistributorLeadKey() + + let operationalStatus: INodeOperationalStatus + switch (statusType) { + case 'normal': { + operationalStatus = { normal: { rationale } } + break + } + case 'noService': { + operationalStatus = { noService: { rationale } } + break + } + case 'noServiceFrom': { + operationalStatus = { + noServiceFrom: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, + } + break + } + case 'noServiceUntil': { + operationalStatus = { + noServiceUntil: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, + } + } + } + + this.log(`Setting node operational status...`, { + bucketId: bucketId.toHuman(), + workerId, + operationalStatus, + }) + + const metadata: ISetNodeOperationalStatus = { + workerId: workerId.toString(), + bucketId: `${bucketId.distributionBucketFamilyId}:${bucketId.distributionBucketIndex}`, + operationalStatus, + } + await this.sendAndFollowTx( + await this.getDecodedPair(leadKey), + this.api.tx.distributionWorkingGroup.leadRemark( + '0x' + Buffer.from(SetNodeOperationalStatus.encode(metadata).finish()).toString('hex') + ) + ) + this.log('Bucket operator metadata successfully set/updated!') + } +} diff --git a/distributor-node/src/commands/operator/set-metadata.ts b/distributor-node/src/commands/operator/set-metadata.ts index e8fae81f3c..f1c0967597 100644 --- a/distributor-node/src/commands/operator/set-metadata.ts +++ b/distributor-node/src/commands/operator/set-metadata.ts @@ -1,8 +1,13 @@ +import { + DistributionBucketOperatorMetadata, + IDistributionBucketOperatorMetadata, + INodeOperationalStatus, + NodeOperationalStatus, +} from '@joystream/metadata-protobuf' import fs from 'fs' import AccountsCommandBase from '../../command-base/accounts' import DefaultCommandBase, { flags } from '../../command-base/default' import { ValidationService } from '../../services/validation/ValidationService' -import { DistributionBucketOperatorMetadata, IDistributionBucketOperatorMetadata } from '@joystream/metadata-protobuf' export default class OperatorSetMetadata extends AccountsCommandBase { static description = `Set/update distribution bucket operator metadata. @@ -22,6 +27,12 @@ export default class OperatorSetMetadata extends AccountsCommandBase { description: 'Root distribution node endpoint', exclusive: ['input'], }), + operationalStatus: flags.enum>({ + char: 'o', + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: false, + description: 'Operational status of the operator', + }), input: flags.string({ char: 'i', description: 'Path to JSON metadata file', @@ -31,19 +42,55 @@ export default class OperatorSetMetadata extends AccountsCommandBase { } async run(): Promise { - const { bucketId, workerId, input, endpoint } = this.parse(OperatorSetMetadata).flags + const { bucketId, workerId, input, endpoint, operationalStatus: statusType } = this.parse(OperatorSetMetadata).flags const workerKey = await this.getDistributorWorkerRoleKey(workerId) + let operationalStatus: INodeOperationalStatus + switch (statusType) { + case 'normal': { + operationalStatus = { normal: {} } + break + } + case 'noService': { + operationalStatus = { noService: {} } + break + } + case 'noServiceFrom': { + operationalStatus = { + noServiceFrom: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, + } + break + } + case 'noServiceUntil': { + operationalStatus = { + noServiceUntil: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, + } + } + } + const validation = new ValidationService() - const metadata: IDistributionBucketOperatorMetadata = input - ? validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) - : { endpoint } + let metadata: IDistributionBucketOperatorMetadata + if (input) { + const params = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) + metadata = { + ...params, + ...(params.operationalStatus && { operationalStatus: params.operationalStatus }), + } + } else { + metadata = { endpoint, operationalStatus } + } this.log(`Setting bucket operator metadata...`, { bucketId: bucketId.toHuman(), workerId, metadata, }) + await this.sendAndFollowTx( await this.getDecodedPair(workerKey), this.api.tx.storage.setDistributionOperatorMetadata( @@ -52,6 +99,6 @@ export default class OperatorSetMetadata extends AccountsCommandBase { '0x' + Buffer.from(DistributionBucketOperatorMetadata.encode(metadata).finish()).toString('hex') ) ) - this.log('Bucket operator metadata succesfully set/updated!') + this.log('Bucket operator metadata successfully set/updated!') } } diff --git a/distributor-node/src/schemas/operatorMetadataSchema.ts b/distributor-node/src/schemas/operatorMetadataSchema.ts index 9bf2fb4d9d..35ea8344cd 100644 --- a/distributor-node/src/schemas/operatorMetadataSchema.ts +++ b/distributor-node/src/schemas/operatorMetadataSchema.ts @@ -21,6 +21,81 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, }, + operationalStatus: { + oneOf: [ + { + type: 'object', + properties: { + normal: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['normal'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noService: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['noService'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceFrom: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + }, + required: ['from'], + }, + }, + required: ['noServiceFrom'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceUntil: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + until: { + type: 'string', + }, + }, + required: ['until'], + }, + }, + required: ['noServiceUntil'], + additionalProperties: false, + }, + ], + }, extra: { type: 'string' }, }, } diff --git a/distributor-node/src/services/networking/NetworkingService.ts b/distributor-node/src/services/networking/NetworkingService.ts index d2d5aba5bf..66c2e3a69f 100644 --- a/distributor-node/src/services/networking/NetworkingService.ts +++ b/distributor-node/src/services/networking/NetworkingService.ts @@ -402,7 +402,7 @@ export class NetworkingService { async checkActiveStorageNodeEndpoints(): Promise { try { - const activeStorageOperators = await this.queryNodeApi.getActiveStorageBucketOperatorsData() + const activeStorageOperators = await this.queryNodeApi.getOperationallyActiveStorageBucketOperatorsData() const endpoints = this.filterStorageNodeEndpoints( activeStorageOperators.map(({ id, operatorMetadata }) => ({ bucketId: id, diff --git a/distributor-node/src/services/networking/query-node/api.ts b/distributor-node/src/services/networking/query-node/api.ts index 6e769ed1f4..3609139fde 100644 --- a/distributor-node/src/services/networking/query-node/api.ts +++ b/distributor-node/src/services/networking/query-node/api.ts @@ -232,11 +232,30 @@ export class QueryNodeApi { return this.getDataObjectsByBagIds(bagIds) } - public getActiveStorageBucketOperatorsData(): Promise { - return this.multipleEntitiesQuery< + public async getOperationallyActiveStorageBucketOperatorsData(): Promise { + const buckets = await this.multipleEntitiesQuery< GetActiveStorageBucketOperatorsDataQuery, GetActiveStorageBucketOperatorsDataQueryVariables >(GetActiveStorageBucketOperatorsData, {}, 'storageBuckets') + + // Filter out nodes/operators under maintenance + return buckets.filter(({ operatorMetadata }) => { + const status = operatorMetadata?.nodeOperationalStatus + const date = new Date() + if ( + !operatorMetadata || + !status || + status.__typename === 'NodeOperationalStatusNormal' || + (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet) + (status.__typename === 'NodeOperationalStatusNoServiceUntil' && + new Date(status.from) > date && + new Date(status.until) < date) // planned future maintenance with end time (which has not started yet) + ) { + return true + } + + return false + }) } public async getPackageVersion(): Promise { diff --git a/distributor-node/src/services/networking/query-node/queries/queries.graphql b/distributor-node/src/services/networking/query-node/queries/queries.graphql index e9c0254852..7a8e97a0cb 100644 --- a/distributor-node/src/services/networking/query-node/queries/queries.graphql +++ b/distributor-node/src/services/networking/query-node/queries/queries.graphql @@ -93,7 +93,7 @@ query getDistributionBucketsWithBagsByIds($ids: [String!]) { } } -query getDistributionBucketsWithBagsByWorkerId($workerId: Int!) { +query getDistributionBucketsWithBagsByWorkerId($workerId: BigInt!) { distributionBuckets(where: { operators_some: { workerId_eq: $workerId, status_eq: ACTIVE } }) { ...DistributionBucketWithBags } @@ -103,6 +103,26 @@ fragment StorageBucketOperatorFields on StorageBucket { id operatorMetadata { nodeEndpoint + nodeOperationalStatus { + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + from + } + ... on NodeOperationalStatusNoServiceUntil { + __typename + forced + from + until + } + } } } diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql index ccae9c01a1..6991db6dcd 100644 --- a/distributor-node/src/services/networking/query-node/schema.graphql +++ b/distributor-node/src/services/networking/query-node/schema.graphql @@ -347,7 +347,7 @@ type DistributionBucketOperator { distributionBucket: DistributionBucket! """ID of the distribution group worker""" - workerId: Int! + workerId: BigInt! """Current operator status""" status: DistributionBucketOperatorStatus! @@ -373,6 +373,9 @@ type DistributionBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -417,6 +420,26 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -462,6 +485,8 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -558,14 +583,14 @@ input DistributionBucketOperatorWhereInput { distributionBucket_isNull: Boolean distributionBucket: DistributionBucketWhereInput workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] status_isNull: Boolean status_eq: DistributionBucketOperatorStatus status_not_eq: DistributionBucketOperatorStatus @@ -651,6 +676,14 @@ input DistributionBucketWhereInput { OR: [DistributionBucketWhereInput!] } +type DistributionNodeOperationalStatusSetEvent { + """Distribution bucket operator""" + bucketOperator: DistributionBucketOperator! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type Event { """{blockNumber}-{indexInBlock}""" id: String! @@ -671,7 +704,7 @@ type Event { data: EventData! } -union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData +union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData | StorageNodeOperationalStatusSetEvent | DistributionNodeOperationalStatusSetEvent input EventDataWhereInput { result_isNull: Boolean @@ -693,6 +726,12 @@ input EventDataWhereInput { dataObjectId_not_startsWith: String dataObjectId_endsWith: String dataObjectId_not_endsWith: String + storageBucket_isNull: Boolean + storageBucket: StorageBucketWhereInput + operationalStatus_isNull: Boolean + operationalStatus: NodeOperationalStatusWhereInput + bucketOperator_isNull: Boolean + bucketOperator: DistributionBucketOperatorWhereInput isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -975,6 +1014,113 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceUntil + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceUntil { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + until: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +input NodeOperationalStatusWhereInput { + rationale_isNull: Boolean + rationale_eq: String + rationale_not_eq: String + rationale_gt: String + rationale_gte: String + rationale_lt: String + rationale_lte: String + rationale_in: [String!] + rationale_not_in: [String!] + rationale_contains: String + rationale_not_contains: String + rationale_containsInsensitive: String + rationale_not_containsInsensitive: String + rationale_startsWith: String + rationale_not_startsWith: String + rationale_endsWith: String + rationale_not_endsWith: String + forced_isNull: Boolean + forced_eq: Boolean + forced_not_eq: Boolean + from_isNull: Boolean + from_eq: DateTime + from_not_eq: DateTime + from_gt: DateTime + from_gte: DateTime + from_lt: DateTime + from_lte: DateTime + from_in: [DateTime!] + from_not_in: [DateTime!] + until_isNull: Boolean + until_eq: DateTime + until_not_eq: DateTime + until_gt: DateTime + until_gte: DateTime + until_lt: DateTime + until_lte: DateTime + until_in: [DateTime!] + until_not_in: [DateTime!] + isTypeOf_isNull: Boolean + isTypeOf_eq: String + isTypeOf_not_eq: String + isTypeOf_gt: String + isTypeOf_gte: String + isTypeOf_lt: String + isTypeOf_lte: String + isTypeOf_in: [String!] + isTypeOf_not_in: [String!] + isTypeOf_contains: String + isTypeOf_not_contains: String + isTypeOf_containsInsensitive: String + isTypeOf_not_containsInsensitive: String + isTypeOf_startsWith: String + isTypeOf_not_startsWith: String + isTypeOf_endsWith: String + isTypeOf_not_endsWith: String +} + type PageInfo { hasNextPage: Boolean! hasPreviousPage: Boolean! @@ -1376,6 +1522,9 @@ type StorageBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -1432,6 +1581,26 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -1477,6 +1646,8 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -1501,12 +1672,12 @@ input StorageBucketOperatorMetadataWhereInput { union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive type StorageBucketOperatorStatusActive { - workerId: Int! + workerId: BigInt! transactorAccountId: String! } type StorageBucketOperatorStatusInvited { - workerId: Int! + workerId: BigInt! } type StorageBucketOperatorStatusMissing { @@ -1524,14 +1695,14 @@ input StorageBucketOperatorStatusWhereInput { phantom_in: [Int!] phantom_not_in: [Int!] workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] transactorAccountId_isNull: Boolean transactorAccountId_eq: String transactorAccountId_not_eq: String @@ -1860,6 +2031,14 @@ input StorageDataObjectWhereInput { OR: [StorageDataObjectWhereInput!] } +type StorageNodeOperationalStatusSetEvent { + """Storage Bucket""" + storageBucket: StorageBucket! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type VideoSubtitle { """(videoId)-{type}-{language}""" id: String! diff --git a/distributor-node/src/services/networking/runtime/api.ts b/distributor-node/src/services/networking/runtime/api.ts index 1398b148ef..59731ddecf 100644 --- a/distributor-node/src/services/networking/runtime/api.ts +++ b/distributor-node/src/services/networking/runtime/api.ts @@ -1,13 +1,13 @@ -import { ApiPromise, WsProvider, SubmittableResult } from '@polkadot/api' -import { SubmittableExtrinsic, AugmentedEvent } from '@polkadot/api/types' +import { ApiPromise, SubmittableResult, WsProvider } from '@polkadot/api' +import { AugmentedEvent, SubmittableExtrinsic } from '@polkadot/api/types' import { KeyringPair } from '@polkadot/keyring/types' import { Balance } from '@polkadot/types/interfaces' -import { formatBalance } from '@polkadot/util' -import { IEvent } from '@polkadot/types/types' import { DispatchError } from '@polkadot/types/interfaces/system' -import { LoggingService } from '../../logging' -import { Logger } from 'winston' import { SpRuntimeDispatchError } from '@polkadot/types/lookup' +import { IEvent } from '@polkadot/types/types' +import { formatBalance } from '@polkadot/util' +import { Logger } from 'winston' +import { LoggingService } from '../../logging' export class ExtrinsicFailedError extends Error {} @@ -30,7 +30,7 @@ export class RuntimeApi { private static async initApi(apiUri: string) { const wsProvider: WsProvider = new WsProvider(apiUri) - const api = await ApiPromise.create({ provider: wsProvider }) + const api = await ApiPromise.create({ provider: wsProvider, throwOnConnect: true }) const [properties, chainType] = await Promise.all([api.rpc.system.properties(), api.rpc.system.chainType()]) diff --git a/joystreamjs/package.json b/joystreamjs/package.json index fd9bdafe5b..90d3a82762 100644 --- a/joystreamjs/package.json +++ b/joystreamjs/package.json @@ -1,6 +1,6 @@ { "name": "@joystream/js", - "version": "1.11.0", + "version": "1.12.0", "license": "GPL-3.0-only", "description": "Joystream JS package provides utilities required to work with Joystream network.", "main": "lib/index.js", diff --git a/metadata-protobuf/package.json b/metadata-protobuf/package.json index 5468693015..aa717cd87d 100644 --- a/metadata-protobuf/package.json +++ b/metadata-protobuf/package.json @@ -1,6 +1,6 @@ { "name": "@joystream/metadata-protobuf", - "version": "2.15.0", + "version": "2.16.0", "description": "Joystream Metadata Protobuf Library", "main": "lib/index.js", "types": "lib/index.d.ts", diff --git a/metadata-protobuf/proto/Storage.proto b/metadata-protobuf/proto/Storage.proto index 9496b751d4..d8407ad356 100644 --- a/metadata-protobuf/proto/Storage.proto +++ b/metadata-protobuf/proto/Storage.proto @@ -15,12 +15,14 @@ message StorageBucketOperatorMetadata { optional string endpoint = 1; // Root storage node endpoint (ie. https://example.com/storage) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator + optional NodeOperationalStatus operational_status = 4; // Node's operational status to set } message DistributionBucketOperatorMetadata { optional string endpoint = 1; // Root distribution node endpoint (ie. https://example.com/distribution) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator + optional NodeOperationalStatus operational_status = 4; // Node's operational status to set } message GeographicalArea { @@ -46,3 +48,40 @@ message DistributionBucketFamilyMetadata { repeated GeographicalArea areas = 3; // Standarized geographical areas covered by the family (providing [{}] will unset the current value) repeated string latency_test_targets = 4; // List of targets (hosts/ips) best suited latency measurements for this family } + +message NodeOperationalStatusNormal { + optional string rationale = 1; + +} + +message NodeOperationalStatusNoService { + optional string rationale = 1; + } + +message NodeOperationalStatusNoServiceFrom { + optional string rationale = 1; + required string from = 2; // date +} + +message NodeOperationalStatusNoServiceUntil { + optional string rationale = 1; + optional string from = 2; // date + required string until = 3; // date +} + + +message NodeOperationalStatus { + oneof node_operational_status { + NodeOperationalStatusNormal normal = 1; + NodeOperationalStatusNoService no_service = 2; + NodeOperationalStatusNoServiceFrom no_service_from = 3; + NodeOperationalStatusNoServiceUntil no_service_until = 4; + } +} + +message SetNodeOperationalStatus { + required string worker_id = 1; // Storage/Distribution Worker ID + required string bucket_id = 2; // Storage/Distribution Bucket ID + required NodeOperationalStatus operational_status = 3; // Node's operational status to set + +} \ No newline at end of file diff --git a/metadata-protobuf/proto/WorkingGroups.proto b/metadata-protobuf/proto/WorkingGroups.proto index 1c2ae68e59..21a25667da 100644 --- a/metadata-protobuf/proto/WorkingGroups.proto +++ b/metadata-protobuf/proto/WorkingGroups.proto @@ -67,6 +67,7 @@ message RemarkMetadataAction { oneof action { ModeratePost moderate_post = 1; VerifyValidator verify_validator = 2; + SetNodeOperationalStatus set_node_operational_status = 3; } } diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md index 0314d63443..fad8c5e615 100644 --- a/storage-node/CHANGELOG.md +++ b/storage-node/CHANGELOG.md @@ -1,3 +1,8 @@ +### 4.2.0 + +- Updates `operator:set-metadata` CLI command to set storage-node's operational status along with other metadata. +- Adds `leader:set-node-operational-status` CLI command to set operational status of any storage-node by Lead. + ### 4.1.1 - Bump deps @polkadot/api v10.7.1 diff --git a/storage-node/README.md b/storage-node/README.md index f86147dbed..4b85d95866 100644 --- a/storage-node/README.md +++ b/storage-node/README.md @@ -6,17 +6,18 @@ Joystream storage node. ![License](https://img.shields.io/github/license/Joystream/joystream) -* [Colossus](#colossus) -* [Description](#description) -* [Installation](#installation) -* [Ubuntu Linux](#ubuntu-linux) -* [Install packages required for installation](#install-packages-required-for-installation) -* [Clone the code repository](#clone-the-code-repository) -* [Install volta](#install-volta) -* [Install project dependencies and build it](#install-project-dependencies-and-build-it) -* [Verify installation](#verify-installation) -* [Usage](#usage) -* [CLI Commands](#cli-commands) + +- [Colossus](#colossus) +- [Description](#description) +- [Installation](#installation) +- [Ubuntu Linux](#ubuntu-linux) +- [Install packages required for installation](#install-packages-required-for-installation) +- [Clone the code repository](#clone-the-code-repository) +- [Install volta](#install-volta) +- [Install project dependencies and build it](#install-project-dependencies-and-build-it) +- [Verify installation](#verify-installation) +- [Usage](#usage) +- [CLI Commands](#cli-commands) # Description @@ -147,29 +148,31 @@ There is also an option to run Colossus as [Docker container](../colossus.Docker # CLI Commands -* [`storage-node help [COMMAND]`](#storage-node-help-command) -* [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) -* [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) -* [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) -* [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) -* [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) -* [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) -* [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) -* [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) -* [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) -* [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) -* [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) -* [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) -* [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) -* [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) -* [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) -* [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) -* [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) -* [`storage-node server`](#storage-node-server) -* [`storage-node util:cleanup`](#storage-node-utilcleanup) -* [`storage-node util:fetch-bucket`](#storage-node-utilfetch-bucket) -* [`storage-node util:multihash`](#storage-node-utilmultihash) -* [`storage-node util:verify-bag-id`](#storage-node-utilverify-bag-id) + +- [`storage-node help [COMMAND]`](#storage-node-help-command) +- [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) +- [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) +- [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) +- [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) +- [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) +- [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) +- [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) +- [`storage-node leader:set-node-operational-status`](#storage-node-leaderset-node-operational-status) +- [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) +- [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) +- [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) +- [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) +- [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) +- [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) +- [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) +- [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) +- [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) +- [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) +- [`storage-node server`](#storage-node-server) +- [`storage-node util:cleanup`](#storage-node-utilcleanup) +- [`storage-node util:fetch-bucket`](#storage-node-utilfetch-bucket) +- [`storage-node util:multihash`](#storage-node-utilmultihash) +- [`storage-node util:verify-bag-id`](#storage-node-utilverify-bag-id) ## `storage-node help [COMMAND]` @@ -213,7 +216,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/cancel-invite.ts)_ +_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/cancel-invite.ts)_ ## `storage-node leader:create-bucket` @@ -244,7 +247,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/create-bucket.ts)_ +_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/create-bucket.ts)_ ## `storage-node leader:delete-bucket` @@ -271,7 +274,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/delete-bucket.ts)_ +_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/delete-bucket.ts)_ ## `storage-node leader:invite-operator` @@ -300,7 +303,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/invite-operator.ts)_ +_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/invite-operator.ts)_ ## `storage-node leader:remove-operator` @@ -327,7 +330,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/remove-operator.ts)_ +_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/remove-operator.ts)_ ## `storage-node leader:set-bucket-limits` @@ -357,7 +360,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/set-bucket-limits.ts)_ +_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-bucket-limits.ts)_ ## `storage-node leader:set-global-uploading-status` @@ -385,7 +388,44 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/set-global-uploading-status.ts)_ +_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-global-uploading-status.ts)_ + +## `storage-node leader:set-node-operational-status` + +Set/update storage node operational status. Requires storage working group leader permissions. + +``` +USAGE + $ storage-node leader:set-node-operational-status + +OPTIONS + -h, --help show CLI help + -i, --bucketId=bucketId (required) Storage bucket ID + -k, --keyFile=keyFile Path to key file to add to the keyring. + -m, --dev Use development mode + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator + + -p, --password=password Password to unlock keyfiles. Multiple + passwords can be passed, to try against all + files. If not specified a single password + can be set in ACCOUNT_PWD environment + variable. + + -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API + URL. Mandatory in non-dev environment. + + -w, --workerId=workerId (required) ID of the operator (storage group + worker) + + -y, --accountUri=accountUri Account URI (optional). If not specified a + single key can be set in ACCOUNT_URI + environment variable. + + --keyStore=keyStore Path to a folder with multiple key files to + load into keystore. +``` + +_See code: [src/commands/leader/set-node-operational-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-node-operational-status.ts)_ ## `storage-node leader:update-bag-limit` @@ -412,7 +452,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bag-limit.ts)_ +_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bag-limit.ts)_ ## `storage-node leader:update-bags` @@ -468,7 +508,7 @@ OPTIONS Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bags.ts)_ +_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bags.ts)_ ## `storage-node leader:update-blacklist` @@ -497,7 +537,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-blacklist.ts)_ +_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-blacklist.ts)_ ## `storage-node leader:update-bucket-status` @@ -526,7 +566,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bucket-status.ts)_ +_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bucket-status.ts)_ ## `storage-node leader:update-data-fee` @@ -553,7 +593,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-data-fee.ts)_ +_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-data-fee.ts)_ ## `storage-node leader:update-data-object-bloat-bond` @@ -581,7 +621,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-data-object-bloat-bond.ts)_ +_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-data-object-bloat-bond.ts)_ ## `storage-node leader:update-dynamic-bag-policy` @@ -611,7 +651,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-dynamic-bag-policy.ts)_ +_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-dynamic-bag-policy.ts)_ ## `storage-node leader:update-voucher-limits` @@ -640,7 +680,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-voucher-limits.ts)_ +_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-voucher-limits.ts)_ ## `storage-node operator:accept-invitation` @@ -673,7 +713,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/operator/accept-invitation.ts)_ +_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/operator/accept-invitation.ts)_ ## `storage-node operator:set-metadata` @@ -684,27 +724,34 @@ USAGE $ storage-node operator:set-metadata OPTIONS - -e, --endpoint=endpoint Root distribution node endpoint - -h, --help show CLI help - -i, --bucketId=bucketId (required) Storage bucket ID - -j, --jsonFile=jsonFile Path to JSON metadata file - -k, --keyFile=keyFile Path to key file to add to the keyring. - -m, --dev Use development mode + -e, --endpoint=endpoint Root distribution node endpoint + -h, --help show CLI help + -i, --bucketId=bucketId (required) Storage bucket ID + -j, --jsonFile=jsonFile Path to JSON metadata file + -k, --keyFile=keyFile Path to key file to add to the keyring. + -m, --dev Use development mode + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator - -p, --password=password Password to unlock keyfiles. Multiple passwords can be passed, to try against all files. - If not specified a single password can be set in ACCOUNT_PWD environment variable. + -p, --password=password Password to unlock keyfiles. Multiple + passwords can be passed, to try against all + files. If not specified a single password + can be set in ACCOUNT_PWD environment + variable. - -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment. + -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API + URL. Mandatory in non-dev environment. - -w, --workerId=workerId (required) Storage operator worker ID + -w, --workerId=workerId (required) Storage operator worker ID - -y, --accountUri=accountUri Account URI (optional). If not specified a single key can be set in ACCOUNT_URI - environment variable. + -y, --accountUri=accountUri Account URI (optional). If not specified a + single key can be set in ACCOUNT_URI + environment variable. - --keyStore=keyStore Path to a folder with multiple key files to load into keystore. + --keyStore=keyStore Path to a folder with multiple key files to + load into keystore. ``` -_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/operator/set-metadata.ts)_ +_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/operator/set-metadata.ts)_ ## `storage-node server` @@ -806,7 +853,7 @@ OPTIONS directory will be used. ``` -_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/server.ts)_ +_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/server.ts)_ ## `storage-node util:cleanup` @@ -844,7 +891,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/cleanup.ts)_ +_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/cleanup.ts)_ ## `storage-node util:fetch-bucket` @@ -877,7 +924,7 @@ OPTIONS under the uploads directory will be used. ``` -_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/fetch-bucket.ts)_ +_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/fetch-bucket.ts)_ ## `storage-node util:multihash` @@ -892,7 +939,7 @@ OPTIONS -h, --help show CLI help ``` -_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/multihash.ts)_ +_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/multihash.ts)_ ## `storage-node util:verify-bag-id` @@ -920,5 +967,6 @@ OPTIONS - dynamic:member:4 ``` -_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/verify-bag-id.ts)_ +_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/verify-bag-id.ts)_ + diff --git a/storage-node/package.json b/storage-node/package.json index f0bfffac5c..65eaf4ae54 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -20,6 +20,7 @@ "@types/base64url": "^2.0.0", "@types/express": "4.17.13", "@types/file-type": "^10.9.1", + "@types/inquirer": "^6.5.0", "@types/lodash": "^4.14.171", "@types/mkdirp": "^0.5.1", "@types/multer": "^1.4.5", @@ -45,6 +46,8 @@ "fast-folder-size": "^1.4.0", "fast-safe-stringify": "^2.1.1", "file-type": "^16.5.0", + "inquirer": "^8.1.2", + "inquirer-datepicker": "2.0.2", "graphql": "^15.3.0", "lodash": "^4.17.21", "mkdirp": "^0.5.1", diff --git a/storage-node/src/@types/inquirer-datepicker/index.d.ts b/storage-node/src/@types/inquirer-datepicker/index.d.ts new file mode 100644 index 0000000000..5f9d649d02 --- /dev/null +++ b/storage-node/src/@types/inquirer-datepicker/index.d.ts @@ -0,0 +1 @@ +declare module 'inquirer-datepicker' diff --git a/storage-node/src/api-spec/openapi.yaml b/storage-node/src/api-spec/openapi.yaml index 4e62b3a197..233032d7f3 100644 --- a/storage-node/src/api-spec/openapi.yaml +++ b/storage-node/src/api-spec/openapi.yaml @@ -278,6 +278,7 @@ components: - downloadBuckets - sync - cleanup + - bucketsOperationalStatuses properties: version: type: string @@ -311,6 +312,21 @@ components: type: array items: type: string + bucketsOperationalStatuses: + type: array + items: + type: object + required: + - 'bucketId' + - 'status' + - 'isForced' + properties: + bucketId: + type: string + status: + type: string + isForced: + type: boolean sync: type: object required: diff --git a/storage-node/src/command-base/ApiCommandBase.ts b/storage-node/src/command-base/ApiCommandBase.ts index 8a803069c2..c9ff367161 100644 --- a/storage-node/src/command-base/ApiCommandBase.ts +++ b/storage-node/src/command-base/ApiCommandBase.ts @@ -1,16 +1,18 @@ +import { JOYSTREAM_ADDRESS_PREFIX } from '@joystream/types' import { Command, flags } from '@oclif/command' -import { createApi } from '../services/runtime/api' -import { addAccountFromJsonFile, addAlicePair, addAccountFromUri } from '../services/runtime/accounts' -import { KeyringPair } from '@polkadot/keyring/types' +import { CLIError } from '@oclif/errors' +import { Input } from '@oclif/parser' import { ApiPromise, Keyring } from '@polkadot/api' +import { KeyringPair } from '@polkadot/keyring/types' import { cryptoWaitReady } from '@polkadot/util-crypto' +import fs from 'fs' +import inquirer, { DistinctQuestion } from 'inquirer' +import inquirerDatepicker from 'inquirer-datepicker' +import path from 'path' import logger from '../services/logger' +import { addAccountFromJsonFile, addAccountFromUri, addAlicePair } from '../services/runtime/accounts' +import { createApi } from '../services/runtime/api' import ExitCodes from './ExitCodes' -import { CLIError } from '@oclif/errors' -import { Input } from '@oclif/parser' -import path from 'path' -import fs from 'fs' -import { JOYSTREAM_ADDRESS_PREFIX } from '@joystream/types' /** * Parent class for all runtime-based commands. Defines common functions. @@ -104,6 +106,8 @@ export default abstract class ApiCommandBase extends Command { } await this.getApi() + + inquirer.registerPrompt('datepicker', inquirerDatepicker) } /** @@ -250,6 +254,21 @@ export default abstract class ApiCommandBase extends Command { return keyring.pairs.filter((pair) => !pair.isLocked).map((pair) => pair.address) } + async datePrompt(question: DistinctQuestion): Promise { + const { result } = await inquirer.prompt([ + { + ...question, + type: 'datepicker', + name: 'result', + clearable: true, + default: new Date().toISOString(), + }, + ]) + + const date = new Date(result) + return date + } + /** * Helper-function for exit after the CLI command. It changes the exit code * depending on the previous extrinsic call success. diff --git a/storage-node/src/commands/leader/set-node-operational-status.ts b/storage-node/src/commands/leader/set-node-operational-status.ts new file mode 100644 index 0000000000..d978cf17e3 --- /dev/null +++ b/storage-node/src/commands/leader/set-node-operational-status.ts @@ -0,0 +1,92 @@ +import { INodeOperationalStatus, NodeOperationalStatus } from '@joystream/metadata-protobuf' +import { flags } from '@oclif/command' +import LeaderCommandBase from '../../command-base/LeaderCommandBase' +import { setStorageNodeOperationalStatus } from '../../services/runtime/extrinsics' + +/** + * CLI command: + * et/update storage node operational status. + * + * @remarks + * Storage working group leader command. Requires storage WG leader privileges. + * Shell command: "leader:set-node-operational-status" + */ +export default class LeadSetNodeOperationalStatus extends LeaderCommandBase { + static description = `Set/update storage node operational status. Requires storage working group leader permissions.` + + static flags = { + bucketId: flags.integer({ + char: 'i', + required: true, + description: 'Storage bucket ID', + }), + workerId: flags.integer({ + char: 'w', + description: 'ID of the operator (storage group worker)', + required: true, + }), + operationalStatus: flags.enum>({ + char: 'o', + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: true, + description: 'Operational status of the operator', + }), + rationale: flags.string({ + char: 'r', + description: 'Rationale for the operational status', + }), + ...LeaderCommandBase.flags, + } + + async run(): Promise { + const { + bucketId, + workerId, + rationale, + operationalStatus: statusType, + } = this.parse(LeadSetNodeOperationalStatus).flags + + const account = this.getAccount() + const api = await this.getApi() + + let operationalStatus: INodeOperationalStatus + switch (statusType) { + case 'normal': { + operationalStatus = { normal: { rationale } } + break + } + case 'noService': { + operationalStatus = { noService: { rationale } } + break + } + case 'noServiceFrom': { + operationalStatus = { + noServiceFrom: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, + } + break + } + case 'noServiceUntil': { + operationalStatus = { + noServiceUntil: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, + } + } + } + + this.log(`Setting node operational status...`, { + bucketId, + workerId, + operationalStatus, + }) + + const success = await setStorageNodeOperationalStatus(api, account, workerId, bucketId, operationalStatus) + + this.exitAfterRuntimeCall(success) + } +} diff --git a/storage-node/src/commands/operator/set-metadata.ts b/storage-node/src/commands/operator/set-metadata.ts index 2e665cacc1..69371371d3 100644 --- a/storage-node/src/commands/operator/set-metadata.ts +++ b/storage-node/src/commands/operator/set-metadata.ts @@ -1,10 +1,14 @@ +import { + INodeOperationalStatus, + IStorageBucketOperatorMetadata, + NodeOperationalStatus, + StorageBucketOperatorMetadata, +} from '@joystream/metadata-protobuf' import { flags } from '@oclif/command' -import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' +import fs from 'fs' import ApiCommandBase from '../../command-base/ApiCommandBase' -import logger from '../../services/logger' import { ValidationService } from '../../services/metadata/validationService' -import { StorageBucketOperatorMetadata, IStorageBucketOperatorMetadata } from '@joystream/metadata-protobuf' -import fs from 'fs' +import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' import { getWorkerRoleAccount } from '../../services/runtime/queries' /** @@ -35,6 +39,12 @@ export default class OperatorSetMetadata extends ApiCommandBase { description: 'Root distribution node endpoint', exclusive: ['jsonFile'], }), + operationalStatus: flags.enum>({ + char: 'o', + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: false, + description: 'Operational status of the operator', + }), jsonFile: flags.string({ char: 'j', description: 'Path to JSON metadata file', @@ -45,16 +55,50 @@ export default class OperatorSetMetadata extends ApiCommandBase { async run(): Promise { const { flags } = this.parse(OperatorSetMetadata) - const { workerId, bucketId, jsonFile, endpoint } = flags + const { workerId, bucketId, jsonFile, endpoint, operationalStatus: statusType } = flags + + let operationalStatus: INodeOperationalStatus + switch (statusType) { + case 'normal': { + operationalStatus = { normal: {} } + break + } + case 'noService': { + operationalStatus = { noService: {} } + break + } + case 'noServiceFrom': { + operationalStatus = { + noServiceFrom: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, + } + break + } + case 'noServiceUntil': { + operationalStatus = { + noServiceUntil: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, + } + } + } const validation = new ValidationService() - const metadata: IStorageBucketOperatorMetadata = jsonFile - ? validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) - : { endpoint } + let metadata: IStorageBucketOperatorMetadata + if (jsonFile) { + const input = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) + metadata = { + ...input, + ...(input.operationalStatus && { operationalStatus: input.operationalStatus }), + } + } else { + metadata = { endpoint, operationalStatus } + } const encodedMetadata = '0x' + Buffer.from(StorageBucketOperatorMetadata.encode(metadata).finish()).toString('hex') - logger.info('Setting the storage operator metadata...') if (flags.dev) { await this.ensureDevelopmentChain() } diff --git a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts index 1e6356647f..5f11551d87 100644 --- a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts +++ b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts @@ -22,6 +22,81 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, }, + operationalStatus: { + oneOf: [ + { + type: 'object', + properties: { + normal: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['normal'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noService: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['noService'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceFrom: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + }, + required: ['from'], + }, + }, + required: ['noServiceFrom'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceUntil: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + until: { + type: 'string', + }, + }, + required: ['until'], + }, + }, + required: ['noServiceUntil'], + additionalProperties: false, + }, + ], + }, extra: { type: 'string' }, }, } diff --git a/storage-node/src/services/metadata/schemas/types.ts b/storage-node/src/services/metadata/schemas/types.ts index f8ae49d234..bd02837513 100644 --- a/storage-node/src/services/metadata/schemas/types.ts +++ b/storage-node/src/services/metadata/schemas/types.ts @@ -1,5 +1,5 @@ -import { schemas } from './schemas' import { OperatorMetadataJson } from '../generated/OperatorMetadataJson' +import { schemas } from './schemas' export type SchemaKey = keyof typeof schemas & string diff --git a/storage-node/src/services/queryNode/api.ts b/storage-node/src/services/queryNode/api.ts index e074905b11..31fd2a026b 100644 --- a/storage-node/src/services/queryNode/api.ts +++ b/storage-node/src/services/queryNode/api.ts @@ -28,6 +28,9 @@ import { GetStorageBucketsByWorkerId, GetStorageBucketsByWorkerIdQuery, GetStorageBucketsByWorkerIdQueryVariables, + GetStorageBucketsOperationalStatus, + GetStorageBucketsOperationalStatusQuery, + GetStorageBucketsOperationalStatusQueryVariables, GetStorageBucketsQuery, GetStorageBucketsQueryVariables, SquidStatus, @@ -179,12 +182,35 @@ export class QueryNodeApi { return result.data[resultKey] } + private getOperationallyActiveStorageBuckets( + buckets: Array + ): Array { + // Filter out nodes/operators under maintenance + return buckets.filter(({ operatorMetadata }) => { + const status = operatorMetadata?.nodeOperationalStatus + const date = new Date() + if ( + !operatorMetadata || + !status || + status.__typename === 'NodeOperationalStatusNormal' || + (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet) + (status.__typename === 'NodeOperationalStatusNoServiceUntil' && + new Date(status.from) > date && + new Date(status.until) < date) // planned future maintenance with end time (which has not started yet) + ) { + return true + } + + return false + }) + } + /** * Returns storage bucket IDs filtered by worker ID. * * @param workerId - worker ID */ - public async getStorageBucketIdsByWorkerId(workerId: number): Promise> { + public async getStorageBucketIdsByWorkerId(workerId: bigint): Promise> { const result = await this.multipleEntitiesQuery< GetStorageBucketsByWorkerIdQuery, GetStorageBucketsByWorkerIdQueryVariables @@ -194,7 +220,7 @@ export class QueryNodeApi { return [] } - return result + return this.getOperationallyActiveStorageBuckets(result) } /** @@ -298,6 +324,25 @@ export class QueryNodeApi { return [] } + return this.getOperationallyActiveStorageBuckets(result) + } + + /** + * Returns storage bucket IDs. + * + */ + public async getStorageBucketsOperationalStatus( + bucketIds: string[] + ): Promise> { + const result = await this.multipleEntitiesQuery< + GetStorageBucketsOperationalStatusQuery, + GetStorageBucketsOperationalStatusQueryVariables + >(GetStorageBucketsOperationalStatus, { ids: bucketIds }, 'storageBuckets') + + if (!result) { + return [] + } + return result } diff --git a/storage-node/src/services/queryNode/queries/queries.graphql b/storage-node/src/services/queryNode/queries/queries.graphql index 8f1d1f254e..bf6930de4a 100644 --- a/storage-node/src/services/queryNode/queries/queries.graphql +++ b/storage-node/src/services/queryNode/queries/queries.graphql @@ -1,5 +1,32 @@ +fragment NodeOperationalStatusFragment on NodeOperationalStatus { + __typename + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + from + } + ... on NodeOperationalStatusNoServiceUntil { + __typename + forced + from + until + } +} + fragment StorageBucketIds on StorageBucket { id + operatorMetadata { + nodeOperationalStatus { + ...NodeOperationalStatusFragment + } + } } query getStorageBuckets { @@ -8,7 +35,7 @@ query getStorageBuckets { } } -query getStorageBucketsByWorkerId($workerId: Int!) { +query getStorageBucketsByWorkerId($workerId: BigInt!) { storageBuckets( where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } } ) { @@ -16,6 +43,34 @@ query getStorageBucketsByWorkerId($workerId: Int!) { } } +query getStorageBucketsOperationalStatus($ids: [String!]) { + storageBuckets(where: { id_in: $ids }) { + id + operatorMetadata { + nodeOperationalStatus { + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + from + } + ... on NodeOperationalStatusNoServiceUntil { + __typename + forced + from + until + } + } + } + } +} + fragment StorageBucketDetails on StorageBucket { id operatorMetadata { diff --git a/storage-node/src/services/queryNode/schema.graphql b/storage-node/src/services/queryNode/schema.graphql index ccae9c01a1..6991db6dcd 100644 --- a/storage-node/src/services/queryNode/schema.graphql +++ b/storage-node/src/services/queryNode/schema.graphql @@ -347,7 +347,7 @@ type DistributionBucketOperator { distributionBucket: DistributionBucket! """ID of the distribution group worker""" - workerId: Int! + workerId: BigInt! """Current operator status""" status: DistributionBucketOperatorStatus! @@ -373,6 +373,9 @@ type DistributionBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -417,6 +420,26 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -462,6 +485,8 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -558,14 +583,14 @@ input DistributionBucketOperatorWhereInput { distributionBucket_isNull: Boolean distributionBucket: DistributionBucketWhereInput workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] status_isNull: Boolean status_eq: DistributionBucketOperatorStatus status_not_eq: DistributionBucketOperatorStatus @@ -651,6 +676,14 @@ input DistributionBucketWhereInput { OR: [DistributionBucketWhereInput!] } +type DistributionNodeOperationalStatusSetEvent { + """Distribution bucket operator""" + bucketOperator: DistributionBucketOperator! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type Event { """{blockNumber}-{indexInBlock}""" id: String! @@ -671,7 +704,7 @@ type Event { data: EventData! } -union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData +union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData | StorageNodeOperationalStatusSetEvent | DistributionNodeOperationalStatusSetEvent input EventDataWhereInput { result_isNull: Boolean @@ -693,6 +726,12 @@ input EventDataWhereInput { dataObjectId_not_startsWith: String dataObjectId_endsWith: String dataObjectId_not_endsWith: String + storageBucket_isNull: Boolean + storageBucket: StorageBucketWhereInput + operationalStatus_isNull: Boolean + operationalStatus: NodeOperationalStatusWhereInput + bucketOperator_isNull: Boolean + bucketOperator: DistributionBucketOperatorWhereInput isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -975,6 +1014,113 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceUntil + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceUntil { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + until: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +input NodeOperationalStatusWhereInput { + rationale_isNull: Boolean + rationale_eq: String + rationale_not_eq: String + rationale_gt: String + rationale_gte: String + rationale_lt: String + rationale_lte: String + rationale_in: [String!] + rationale_not_in: [String!] + rationale_contains: String + rationale_not_contains: String + rationale_containsInsensitive: String + rationale_not_containsInsensitive: String + rationale_startsWith: String + rationale_not_startsWith: String + rationale_endsWith: String + rationale_not_endsWith: String + forced_isNull: Boolean + forced_eq: Boolean + forced_not_eq: Boolean + from_isNull: Boolean + from_eq: DateTime + from_not_eq: DateTime + from_gt: DateTime + from_gte: DateTime + from_lt: DateTime + from_lte: DateTime + from_in: [DateTime!] + from_not_in: [DateTime!] + until_isNull: Boolean + until_eq: DateTime + until_not_eq: DateTime + until_gt: DateTime + until_gte: DateTime + until_lt: DateTime + until_lte: DateTime + until_in: [DateTime!] + until_not_in: [DateTime!] + isTypeOf_isNull: Boolean + isTypeOf_eq: String + isTypeOf_not_eq: String + isTypeOf_gt: String + isTypeOf_gte: String + isTypeOf_lt: String + isTypeOf_lte: String + isTypeOf_in: [String!] + isTypeOf_not_in: [String!] + isTypeOf_contains: String + isTypeOf_not_contains: String + isTypeOf_containsInsensitive: String + isTypeOf_not_containsInsensitive: String + isTypeOf_startsWith: String + isTypeOf_not_startsWith: String + isTypeOf_endsWith: String + isTypeOf_not_endsWith: String +} + type PageInfo { hasNextPage: Boolean! hasPreviousPage: Boolean! @@ -1376,6 +1522,9 @@ type StorageBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -1432,6 +1581,26 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -1477,6 +1646,8 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -1501,12 +1672,12 @@ input StorageBucketOperatorMetadataWhereInput { union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive type StorageBucketOperatorStatusActive { - workerId: Int! + workerId: BigInt! transactorAccountId: String! } type StorageBucketOperatorStatusInvited { - workerId: Int! + workerId: BigInt! } type StorageBucketOperatorStatusMissing { @@ -1524,14 +1695,14 @@ input StorageBucketOperatorStatusWhereInput { phantom_in: [Int!] phantom_not_in: [Int!] workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] transactorAccountId_isNull: Boolean transactorAccountId_eq: String transactorAccountId_not_eq: String @@ -1860,6 +2031,14 @@ input StorageDataObjectWhereInput { OR: [StorageDataObjectWhereInput!] } +type StorageNodeOperationalStatusSetEvent { + """Storage Bucket""" + storageBucket: StorageBucket! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type VideoSubtitle { """(videoId)-{type}-{language}""" id: String! diff --git a/storage-node/src/services/runtime/extrinsics.ts b/storage-node/src/services/runtime/extrinsics.ts index 924009f4f8..4aef7d2a03 100644 --- a/storage-node/src/services/runtime/extrinsics.ts +++ b/storage-node/src/services/runtime/extrinsics.ts @@ -1,3 +1,8 @@ +import { + INodeOperationalStatus, + ISetNodeOperationalStatus, + SetNodeOperationalStatus, +} from '@joystream/metadata-protobuf' import { ApiPromise } from '@polkadot/api' import { KeyringPair } from '@polkadot/keyring/types' import { PalletStorageBagIdType as BagId, PalletStorageDynamicBagType as DynamicBagType } from '@polkadot/types/lookup' @@ -576,3 +581,37 @@ export async function updateBlacklist( return sendAndFollowNamedTx(api, account, tx) }) } + +/** + * Set/update storage nodes operational status by Lead + * + * @remarks + * It sends an lead remark extrinsic to the runtime. + * + * @param api - runtime API promise + * @param account - KeyringPair instance + * @param workerId - Worker ID + * @param buckerId - Bucket ID + * @param operationalStatus - Operational Status to set for the node + * @returns promise with a success flag. + */ +export async function setStorageNodeOperationalStatus( + api: ApiPromise, + account: KeyringPair, + workerId: number, + bucketId: number, + operationalStatus: INodeOperationalStatus +): Promise { + return await extrinsicWrapper(() => { + const metadata: ISetNodeOperationalStatus = { + workerId: workerId.toString(), + bucketId: bucketId.toString(), + operationalStatus, + } + const tx = api.tx.storageWorkingGroup.leadRemark( + '0x' + Buffer.from(SetNodeOperationalStatus.encode(metadata).finish()).toString('hex') + ) + + return sendAndFollowNamedTx(api, account, tx) + }) +} diff --git a/storage-node/src/services/sync/storageObligations.ts b/storage-node/src/services/sync/storageObligations.ts index 86fa98ec15..ff51a7691e 100644 --- a/storage-node/src/services/sync/storageObligations.ts +++ b/storage-node/src/services/sync/storageObligations.ts @@ -130,7 +130,7 @@ export async function getStorageObligationsFromRuntime( * @returns storage bucket IDs */ export async function getStorageBucketIdsByWorkerId(qnApi: QueryNodeApi, workerId: number): Promise { - const idFragments = await qnApi.getStorageBucketIdsByWorkerId(workerId) + const idFragments = await qnApi.getStorageBucketIdsByWorkerId(BigInt(workerId)) const ids = idFragments.map((frag) => frag.id) return ids diff --git a/storage-node/src/services/webApi/controllers/stateApi.ts b/storage-node/src/services/webApi/controllers/stateApi.ts index 10273719f4..d42a2070c2 100644 --- a/storage-node/src/services/webApi/controllers/stateApi.ts +++ b/storage-node/src/services/webApi/controllers/stateApi.ts @@ -147,6 +147,18 @@ export async function getStatus(req: express.Request, res: express.Response ({ + bucketId: id, + status: operatorMetadata?.nodeOperationalStatus?.__typename || 'NodeOperationalStatusNormal', + isForced: + operatorMetadata?.nodeOperationalStatus?.__typename === 'NodeOperationalStatusNormal' + ? false + : operatorMetadata?.nodeOperationalStatus?.forced || false, + }) + ) + res.status(200).json({ version: proc.version, uptime: Math.floor(process.uptime()), @@ -155,6 +167,7 @@ export async function getStatus(req: express.Request, res: express.Response