From 8a10f93a9fad694d33d0ed5d32c6249f4f0c4733 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 10:07:26 +0500 Subject: [PATCH 01/18] add protobuf schemas for operational status feat --- metadata-protobuf/proto/Metaprotocol.proto | 12 +++++++++ metadata-protobuf/proto/Storage.proto | 31 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/metadata-protobuf/proto/Metaprotocol.proto b/metadata-protobuf/proto/Metaprotocol.proto index 04a29c1470..0757eddffc 100644 --- a/metadata-protobuf/proto/Metaprotocol.proto +++ b/metadata-protobuf/proto/Metaprotocol.proto @@ -188,3 +188,15 @@ message ChannelOwnerRemarked { ModerateComment moderate_comment = 5; } } + + +message LeadRemarked { + // lead_remark extrinsic would emit event containing + // any one of the following serialized messages + oneof lead_remarked { + ModeratePost moderate_post = 1; + SetNodeOperationalStatus set_node_operational_status = 2; + // Other lead remark messages + // ... + } + } \ No newline at end of file diff --git a/metadata-protobuf/proto/Storage.proto b/metadata-protobuf/proto/Storage.proto index 9496b751d4..93bece2cbd 100644 --- a/metadata-protobuf/proto/Storage.proto +++ b/metadata-protobuf/proto/Storage.proto @@ -15,12 +15,15 @@ message StorageBucketOperatorMetadata { optional string endpoint = 1; // Root storage node endpoint (ie. https://example.com/storage) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator + optional NodeOperationalStatusMetadata operational_status = 4; // Node's operational status to set + } message DistributionBucketOperatorMetadata { optional string endpoint = 1; // Root distribution node endpoint (ie. https://example.com/distribution) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator + optional NodeOperationalStatusMetadata operational_status = 4; // Node's operational status to set } message GeographicalArea { @@ -46,3 +49,31 @@ message DistributionBucketFamilyMetadata { repeated GeographicalArea areas = 3; // Standarized geographical areas covered by the family (providing [{}] will unset the current value) repeated string latency_test_targets = 4; // List of targets (hosts/ips) best suited latency measurements for this family } + +message NodeOperationalStatusMetadata { + enum OperationalStatus { + // Node is operating normally + NORMAL = 0; + + // Node is not operational + NO_SERVICE = 1; + } + + // Node's Operational status to set + optional OperationalStatus status = 1; + + // Date (ISO format) from which the node won't be operational (Should be set if status is NoServiceFrom or NoServiceDuring) + optional string no_service_from = 2; + + // Date (ISO format) until which the node won't be operational (Should be set if status is NoServiceDuring) + optional string no_service_to = 3; + + // Rationale for setting the current status + optional string rationale = 4; +} + +message SetNodeOperationalStatus { + optional NodeOperationalStatusMetadata operational_status = 1; // Node's operational status to set + optional string worker_id = 2; // Storage/Distribution Worker ID + optional string bucket_id = 3; // Storage/Distribution Bucket ID +} From bb0b5f512e7afd9d119643cd80ac11198a199025 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 10:19:11 +0500 Subject: [PATCH 02/18] add QN schemas for operational status feat --- query-node/schemas/storage.graphql | 50 ++++++++++++++++++++++ query-node/schemas/storageEvents.graphql | 53 ++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 query-node/schemas/storageEvents.graphql diff --git a/query-node/schemas/storage.graphql b/query-node/schemas/storage.graphql index 77e005133e..73b39351a0 100644 --- a/query-node/schemas/storage.graphql +++ b/query-node/schemas/storage.graphql @@ -58,6 +58,50 @@ type NodeLocationMetadata @entity { coordinates: GeoCoordinates } +type NodeOperationalStatusNormal @variant { + "Reason why node was set to this state" + rationale: String +} + +type NodeOperationalStatusNoService @variant { + "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." + forced: Boolean! + + "Reason why node was set to this state" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom @variant { + "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." + forced: Boolean! + + "The time from which the bucket would have to no service" + from: DateTime! + + "Reason why node was set to this state" + rationale: String +} + +type NodeOperationalStatusNoServiceDuring @variant { + "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." + forced: Boolean! + + "The time from which the bucket would have to no service" + from: DateTime! + + "The time until which the bucket would have to no service" + to: DateTime! + + "Reason why node was set to this state" + rationale: String +} + +union NodeOperationalStatus = + NodeOperationalStatusNormal + | NodeOperationalStatusNoService + | NodeOperationalStatusNoServiceFrom + | NodeOperationalStatusNoServiceDuring + type StorageBucketOperatorMetadata @entity { "Root node endpoint" nodeEndpoint: String @@ -65,6 +109,9 @@ type StorageBucketOperatorMetadata @entity { "Optional node location metadata" nodeLocation: NodeLocationMetadata + "Optional node operational status" + nodeOperationalStatus: NodeOperationalStatus + "Additional information about the node/operator" extra: String } @@ -258,6 +305,9 @@ type DistributionBucketOperatorMetadata @entity { "Optional node location metadata" nodeLocation: NodeLocationMetadata + "Optional node operational status" + nodeOperationalStatus: NodeOperationalStatus + "Additional information about the node/operator" extra: String } diff --git a/query-node/schemas/storageEvents.graphql b/query-node/schemas/storageEvents.graphql new file mode 100644 index 0000000000..724e28e08f --- /dev/null +++ b/query-node/schemas/storageEvents.graphql @@ -0,0 +1,53 @@ +type StorageNodeOperationalStatusSetEvent implements Event @entity { + ### GENERIC DATA ### + + "(network}-{blockNumber}-{indexInBlock}" + id: ID! + + "Hash of the extrinsic which caused the event to be emitted" + inExtrinsic: String + + "Blocknumber of the block in which the event was emitted." + inBlock: Int! + + "Network the block was produced in" + network: Network! + + "Index of event in block from which it was emitted." + indexInBlock: Int! + + ### SPECIFIC DATA ### + + "Storage Bucket" + storageBucket: StorageBucket + + "Operational status that was set" + operationalStatus: NodeOperationalStatus! +} + +type DistributionNodeOperationalStatusSetEvent implements Event @entity { + ### GENERIC DATA ### + + "(network}-{blockNumber}-{indexInBlock}" + id: ID! + + "Hash of the extrinsic which caused the event to be emitted" + inExtrinsic: String + + "Blocknumber of the block in which the event was emitted." + inBlock: Int! + + "Network the block was produced in" + network: Network! + + "Index of event in block from which it was emitted." + indexInBlock: Int! + + ### SPECIFIC DATA ### + + "Distribution bucket operator" + bucketOperator: DistributionBucketOperator + + "Operational status that was set" + operationalStatus: NodeOperationalStatus! +} From 407cdeb433bac0e77082b24dc2cd69645dfbe9a9 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 10:50:03 +0500 Subject: [PATCH 03/18] add QN mappings for operational status feat. --- query-node/mappings/src/storage/index.ts | 174 +++++++++++++++----- query-node/mappings/src/storage/metadata.ts | 95 +++++++++-- query-node/mappings/src/workingGroups.ts | 168 ++++++++++--------- 3 files changed, 304 insertions(+), 133 deletions(-) diff --git a/query-node/mappings/src/storage/index.ts b/query-node/mappings/src/storage/index.ts index 8086a2ac22..b6a2920304 100644 --- a/query-node/mappings/src/storage/index.ts +++ b/query-node/mappings/src/storage/index.ts @@ -1,7 +1,29 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common' +import { DatabaseManager, EventContext, StoreContext, SubstrateEvent } from '@joystream/hydra-common' +import { ISetNodeOperationalStatus } from '@joystream/metadata-protobuf' +import { isSet } from '@joystream/metadata-protobuf/utils' +import BN from 'bn.js' +import { + DistributionBucket, + DistributionBucketFamily, + DistributionBucketOperator, + DistributionBucketOperatorMetadata, + DistributionBucketOperatorStatus, + DistributionNodeOperationalStatusSetEvent, + GeoCoordinates, + NodeLocationMetadata, + StorageBag, + StorageBucket, + StorageBucketOperatorMetadata, + StorageBucketOperatorStatusActive, + StorageBucketOperatorStatusInvited, + StorageBucketOperatorStatusMissing, + StorageDataObject, + StorageNodeOperationalStatusSetEvent, + WorkingGroup, +} from 'query-node/dist/model' import { Storage_DataObjectsDeletedEvent_V1001 as DataObjectsDeletedEvent_V1001, Storage_DataObjectsMovedEvent_V1001 as DataObjectsMovedEvent_V1001, @@ -9,70 +31,62 @@ import { Storage_DataObjectsUploadedEvent_V1001 as DataObjectsUploadedEvent_V1001, Storage_DistributionBucketCreatedEvent_V1001 as DistributionBucketCreatedEvent_V1001, Storage_DistributionBucketDeletedEvent_V1001 as DistributionBucketDeletedEvent_V1001, - Storage_DistributionBucketsUpdatedForBagEvent_V1001 as DistributionBucketsUpdatedForBagEvent_V1001, Storage_DistributionBucketFamilyCreatedEvent_V1001 as DistributionBucketFamilyCreatedEvent_V1001, Storage_DistributionBucketFamilyDeletedEvent_V1001 as DistributionBucketFamilyDeletedEvent_V1001, Storage_DistributionBucketFamilyMetadataSetEvent_V1001 as DistributionBucketFamilyMetadataSetEvent_V1001, - Storage_DistributionBucketMetadataSetEvent_V1001 as DistributionBucketMetadataSetEvent_V1001, Storage_DistributionBucketInvitationAcceptedEvent_V1001 as DistributionBucketInvitationAcceptedEvent_V1001, Storage_DistributionBucketInvitationCancelledEvent_V1001 as DistributionBucketInvitationCancelledEvent_V1001, - Storage_StorageOperatorMetadataSetEvent_V1001 as StorageOperatorMetadataSetEvent_V1001, - Storage_StorageBucketCreatedEvent_V1001 as StorageBucketCreatedEvent_V1001, - Storage_StorageBucketDeletedEvent_V1001 as StorageBucketDeletedEvent_V1001, - Storage_StorageBucketInvitationAcceptedEvent_V1001 as StorageBucketInvitationAcceptedEvent_V1001, - Storage_StorageBucketInvitationCancelledEvent_V1001 as StorageBucketInvitationCancelledEvent_V1001, + Storage_DistributionBucketMetadataSetEvent_V1001 as DistributionBucketMetadataSetEvent_V1001, Storage_DistributionBucketModeUpdatedEvent_V1001 as DistributionBucketModeUpdatedEvent_V1001, - Storage_DynamicBagCreatedEvent_V1001 as DynamicBagCreatedEvent_V1001, - Storage_DynamicBagDeletedEvent_V1001 as DynamicBagDeletedEvent_V1001, - Storage_VoucherChangedEvent_V1001 as VoucherChangedEvent_V1001, Storage_DistributionBucketOperatorInvitedEvent_V1001 as DistributionBucketOperatorInvitedEvent_V1001, Storage_DistributionBucketOperatorRemovedEvent_V1001 as DistributionBucketOperatorRemovedEvent_V1001, Storage_DistributionBucketStatusUpdatedEvent_V1001 as DistributionBucketStatusUpdatedEvent_V1001, + Storage_DistributionBucketsUpdatedForBagEvent_V1001 as DistributionBucketsUpdatedForBagEvent_V1001, + Storage_DynamicBagCreatedEvent_V1001 as DynamicBagCreatedEvent_V1001, + Storage_DynamicBagDeletedEvent_V1001 as DynamicBagDeletedEvent_V1001, Storage_PendingDataObjectsAcceptedEvent_V1001 as PendingDataObjectsAcceptedEvent_V1001, + Storage_StorageBucketCreatedEvent_V1001 as StorageBucketCreatedEvent_V1001, + Storage_StorageBucketDeletedEvent_V1001 as StorageBucketDeletedEvent_V1001, + Storage_StorageBucketInvitationAcceptedEvent_V1001 as StorageBucketInvitationAcceptedEvent_V1001, + Storage_StorageBucketInvitationCancelledEvent_V1001 as StorageBucketInvitationCancelledEvent_V1001, Storage_StorageBucketOperatorInvitedEvent_V1001 as StorageBucketOperatorInvitedEvent_V1001, Storage_StorageBucketStatusUpdatedEvent_V1001 as StorageBucketStatusUpdatedEvent_V1001, Storage_StorageBucketVoucherLimitsSetEvent_V1001 as StorageBucketVoucherLimitsSetEvent_V1001, Storage_StorageBucketsUpdatedForBagEvent_V1001 as StorageBucketsUpdatedForBagEvent_V1001, + Storage_StorageOperatorMetadataSetEvent_V1001 as StorageOperatorMetadataSetEvent_V1001, + Storage_VoucherChangedEvent_V1001 as VoucherChangedEvent_V1001, } from '../../generated/types' import { - DistributionBucket, - DistributionBucketFamily, - DistributionBucketOperator, - DistributionBucketOperatorMetadata, - DistributionBucketOperatorStatus, - NodeLocationMetadata, - StorageBag, - StorageBucket, - StorageBucketOperatorStatusActive, - StorageBucketOperatorStatusInvited, - StorageBucketOperatorStatusMissing, - StorageDataObject, - GeoCoordinates, -} from 'query-node/dist/model' -import BN from 'bn.js' -import { getById, inconsistentState } from '../common' -import { videoRelationsForCounters, unsetAssetRelations } from '../content/utils' + deterministicEntityId, + genericEventFields, + getById, + getWorker, + inconsistentState, + invalidMetadata, +} from '../common' +import { unsetAssetRelations, videoRelationsForCounters } from '../content/utils' +import { getAllManagers } from '../derivedPropertiesManager/applications' import { processDistributionBucketFamilyMetadata, processDistributionOperatorMetadata, + processNodeOperationalStatusMetadata, processStorageOperatorMetadata, } from './metadata' import { createDataObjects, - getStorageBucketWithOperatorMetadata, + deleteDataObjects, + distributionBucketId, + distributionBucketIdByFamilyAndIndex, + distributionOperatorId, getBag, - getDynamicBagId, - getDynamicBagOwner, getDataObjectsInBag, - getDynamicBag, getDistributionBucketFamilyWithMetadata, getDistributionBucketOperatorWithMetadata, - distributionBucketId, - distributionOperatorId, - distributionBucketIdByFamilyAndIndex, - deleteDataObjects, + getDynamicBag, + getDynamicBagId, + getDynamicBagOwner, + getStorageBucketWithOperatorMetadata, } from './utils' -import { getAllManagers } from '../derivedPropertiesManager/applications' // STORAGE BUCKETS @@ -585,3 +599,89 @@ async function removeDistributionBucketOperator(store: DatabaseManager, operator } } } + +export async function processSetNodeOperationalStatusMessage( + store: DatabaseManager, + event: SubstrateEvent, + workingGroup: WorkingGroup, + meta: ISetNodeOperationalStatus +): Promise { + if (workingGroup.name !== 'distributionWorkingGroup' && workingGroup.name !== 'storageWorkingGroup') { + return invalidMetadata(`The ${workingGroup.name} is incompatible with the remarked setNodeOperationalStatus`) + } + + const workerId = Number(meta.workerId) + const bucketId = String(meta.bucketId) + + if ((await getWorker(store, workingGroup.name, workerId)) === undefined) { + return invalidMetadata(`The worker ${workerId} does not exist in the ${workingGroup.name} working group`) + } + + // Update the operational status of Storage node + if (workingGroup.name === 'storageWorkingGroup') { + const storageBucket = await store.get(StorageBucket, { where: { id: bucketId }, relations: ['operatorMetadata'] }) + if (!storageBucket) { + return invalidMetadata(`The storage bucket ${bucketId} does not exist`) + } else if (storageBucket.operatorStatus.isTypeOf !== 'StorageBucketOperatorStatusActive') { + return invalidMetadata(`The storage bucket ${bucketId} is not active`) + } + + // create metadata entity if it does not exist already + const metadataEntity = + storageBucket.operatorMetadata || new StorageBucketOperatorMetadata({ id: deterministicEntityId(event) }) + + if (isSet(meta.operationalStatus)) { + metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( + 'lead', + metadataEntity.nodeOperationalStatus, + meta.operationalStatus + ) + } + await store.save(metadataEntity) + + // event processing + + const operationalStatusSetEvent = new StorageNodeOperationalStatusSetEvent({ + ...genericEventFields(event), + storageBucket, + operationalStatus: metadataEntity.nodeOperationalStatus, + }) + + await store.save(operationalStatusSetEvent) + } + + // Update the operational status of Distribution node + if (workingGroup.name === 'distributionWorkingGroup') { + const distributionOperatorId = `${bucketId}-${workerId}` + const operator = await store.get(DistributionBucketOperator, { + where: { id: distributionOperatorId }, + relations: ['metadata'], + }) + if (!operator) { + return invalidMetadata(`The distribution bucket operator ${distributionOperatorId} does not exist`) + } + + // create metadata entity if it does not exist already + const metadataEntity = + operator.metadata || new DistributionBucketOperatorMetadata({ id: deterministicEntityId(event) }) + + if (isSet(meta.operationalStatus)) { + metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( + 'lead', + metadataEntity.nodeOperationalStatus, + meta.operationalStatus + ) + } + await store.save(metadataEntity) + + // event processing + + const operationalStatusSetEvent = new DistributionNodeOperationalStatusSetEvent({ + ...genericEventFields(event), + bucketOperator: operator, + operationalStatus: metadataEntity.nodeOperationalStatus, + }) + + await store.save(operationalStatusSetEvent) + } +} diff --git a/query-node/mappings/src/storage/metadata.ts b/query-node/mappings/src/storage/metadata.ts index f18fe29724..6d7a8f8f5a 100644 --- a/query-node/mappings/src/storage/metadata.ts +++ b/query-node/mappings/src/storage/metadata.ts @@ -1,27 +1,34 @@ import { DatabaseManager, SubstrateEvent } from '@joystream/hydra-common' import { + DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto, + DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto, + GeographicalArea as GeographicalAreaProto, + INodeLocationMetadata, + INodeOperationalStatusMetadata, + NodeOperationalStatusMetadata, + StorageBucketOperatorMetadata as StorageBucketOperatorMetadataProto, +} from '@joystream/metadata-protobuf' +import { isEmptyObject, isSet, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils' +import { Bytes } from '@polkadot/types' +import _ from 'lodash' +import { + Continent, + DistributionBucketFamilyGeographicArea, DistributionBucketFamilyMetadata, DistributionBucketOperatorMetadata, - StorageBucketOperatorMetadata, GeoCoordinates, - NodeLocationMetadata, - Continent, GeographicalAreaContinent, GeographicalAreaCountry, GeographicalAreaSubdivistion, - DistributionBucketFamilyGeographicArea, + NodeLocationMetadata, + NodeOperationalStatus, + NodeOperationalStatusNoService, + NodeOperationalStatusNoServiceDuring, + NodeOperationalStatusNoServiceFrom, + NodeOperationalStatusNormal, + StorageBucketOperatorMetadata, } from 'query-node/dist/model' import { deserializeMetadata, deterministicEntityId, invalidMetadata } from '../common' -import { Bytes } from '@polkadot/types' -import { - DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto, - StorageBucketOperatorMetadata as StorageBucketOperatorMetadataProto, - DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto, - INodeLocationMetadata, - GeographicalArea as GeographicalAreaProto, -} from '@joystream/metadata-protobuf' -import { isSet, isEmptyObject, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils' -import _ from 'lodash' const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Continent]: Continent } = { [GeographicalAreaProto.Continent.AF]: Continent.AF, @@ -66,6 +73,52 @@ async function processNodeLocationMetadata( return nodeLocation } +export function processNodeOperationalStatusMetadata( + actorContext: 'lead' | 'worker', + current: typeof NodeOperationalStatus | undefined, + meta: INodeOperationalStatusMetadata +): typeof NodeOperationalStatus { + const isCurrentForced = + current && + (current instanceof NodeOperationalStatusNoService || + current instanceof NodeOperationalStatusNoServiceFrom || + current instanceof NodeOperationalStatusNoServiceDuring) && + current.forced + + // if current state is forced by lead, then prevent worker from unilaterally reversing. + if (isCurrentForced && actorContext === 'worker') { + return current + } + + // set node state to NoService + if (meta.status === NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE) { + if (meta.noServiceFrom && meta.noServiceTo) { + const status = new NodeOperationalStatusNoServiceDuring() + status.from = new Date(meta.noServiceFrom) + status.to = new Date(meta.noServiceTo) + status.rationale = meta.rationale || (null as any) + status.forced = actorContext === 'lead' + return status + } else if (meta.noServiceFrom && !meta.noServiceTo) { + const status = new NodeOperationalStatusNoServiceFrom() + status.from = new Date(meta.noServiceFrom) + status.rationale = meta.rationale || (null as any) + status.forced = actorContext === 'lead' + return status + } else if (!meta.noServiceFrom && !meta.noServiceTo) { + const status = new NodeOperationalStatusNoService() + status.rationale = meta.rationale || (null as any) + status.forced = actorContext === 'lead' + return status + } + } + + // Default operational status of the node + const status = new NodeOperationalStatusNormal() + status.rationale = meta.rationale || (null as any) + return status +} + export async function processDistributionOperatorMetadata( event: SubstrateEvent, store: DatabaseManager, @@ -88,6 +141,13 @@ export async function processDistributionOperatorMetadata( if (isSet(meta.extra)) { metadataEntity.extra = meta.extra } + if (isSet(meta.operationalStatus)) { + metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( + 'worker', + metadataEntity.nodeOperationalStatus, + meta.operationalStatus + ) + } await store.save(metadataEntity) @@ -116,6 +176,13 @@ export async function processStorageOperatorMetadata( if (isSet(meta.extra)) { metadataEntity.extra = meta.extra || (null as any) } + if (isSet(meta.operationalStatus)) { + metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( + 'worker', + metadataEntity.nodeOperationalStatus, + meta.operationalStatus + ) + } await store.save(metadataEntity) diff --git a/query-node/mappings/src/workingGroups.ts b/query-node/mappings/src/workingGroups.ts index 4e3b441c0b..c7daea7eda 100644 --- a/query-node/mappings/src/workingGroups.ts +++ b/query-node/mappings/src/workingGroups.ts @@ -1,11 +1,84 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { EventContext, StoreContext, DatabaseManager, SubstrateEvent, FindOneOptions } from '@joystream/hydra-common' +import { DatabaseManager, EventContext, FindOneOptions, StoreContext, SubstrateEvent } from '@joystream/hydra-common' +import { + ApplicationMetadata, + IAddUpcomingOpening, + IOpeningMetadata, + IRemoveUpcomingOpening, + ISetGroupMetadata, + IWorkingGroupMetadata, + IWorkingGroupMetadataAction, + LeadRemarked, + OpeningMetadata, + RemarkMetadataAction, + WorkingGroupMetadataAction, +} from '@joystream/metadata-protobuf' +import { DecodedMetadataObject } from '@joystream/metadata-protobuf/types' +import { isSet } from '@joystream/metadata-protobuf/utils' +import { createType } from '@joystream/types' +import { Bytes } from '@polkadot/types' +import BN from 'bn.js' +import { + ApplicationFormQuestion, + ApplicationFormQuestionAnswer, + ApplicationFormQuestionType, + ApplicationStatusAccepted, + ApplicationStatusCancelled, + ApplicationStatusPending, + ApplicationStatusRejected, + ApplicationStatusWithdrawn, + ApplicationWithdrawnEvent, + AppliedOnOpeningEvent, + BudgetFundedEvent, + BudgetSetEvent, + BudgetSpendingEvent, + InvalidActionMetadata, + LeaderSetEvent, + LeaderUnsetEvent, + Membership, + NewMissedRewardLevelReachedEvent, + OpeningAddedEvent, + // LeaderSetEvent, + OpeningCanceledEvent, + OpeningFilledEvent, + OpeningStatusCancelled, + OpeningStatusFilled, + OpeningStatusOpen, + RewardPaidEvent, + RewardPaymentType, + StakeDecreasedEvent, + StakeIncreasedEvent, + StakeSlashedEvent, + StatusTextChangedEvent, + TerminatedLeaderEvent, + TerminatedWorkerEvent, + UpcomingOpeningAdded, + UpcomingOpeningRemoved, + UpcomingWorkingGroupOpening, + Worker, + WorkerExitedEvent, + WorkerRewardAccountUpdatedEvent, + WorkerRewardAmountUpdatedEvent, + WorkerRoleAccountUpdatedEvent, + WorkerStartedLeavingEvent, + WorkerStatusActive, + WorkerStatusLeaving, + WorkerStatusLeft, + WorkerStatusTerminated, + WorkingGroup, + WorkingGroupApplication, + WorkingGroupMetadata, + WorkingGroupMetadataActionResult, + WorkingGroupMetadataSet, + WorkingGroupOpening, + WorkingGroupOpeningMetadata, + WorkingGroupOpeningType, +} from 'query-node/dist/model' import { StorageWorkingGroup_BudgetSetEvent_V1001, - StorageWorkingGroup_WorkerRewardAmountUpdatedEvent_V1001 as WorkingGroup_WorkerRewardAmountUpdatedEvent_V1001, StorageWorkingGroup_ApplicationWithdrawnEvent_V1001 as WorkingGroup_ApplicationWithdrawnEvent_V1001, StorageWorkingGroup_AppliedOnOpeningEvent_V1001 as WorkingGroup_AppliedOnOpeningEvent_V1001, StorageWorkingGroup_BudgetSpendingEvent_V1001 as WorkingGroup_BudgetSpendingEvent_V1001, @@ -23,97 +96,26 @@ import { StorageWorkingGroup_WorkerExitedEvent_V1001 as WorkingGroup_WorkerExitedEvent_V1001, StorageWorkingGroup_WorkerRemarkedEvent_V1001 as WorkingGroup_WorkerRemarkedEvent_V1001, StorageWorkingGroup_WorkerRewardAccountUpdatedEvent_V1001 as WorkingGroup_WorkerRewardAccountUpdatedEvent_V1001, + StorageWorkingGroup_WorkerRewardAmountUpdatedEvent_V1001 as WorkingGroup_WorkerRewardAmountUpdatedEvent_V1001, StorageWorkingGroup_WorkerRoleAccountUpdatedEvent_V1001 as WorkingGroup_WorkerRoleAccountUpdatedEvent_V1001, StorageWorkingGroup_WorkerStartedLeavingEvent_V1001 as WorkingGroup_WorkerStartedLeavingEvent_V1001, StorageWorkingGroup_WorkingGroupBudgetFundedEvent_V1001 as WorkingGroup_WorkingGroupBudgetFundedEvent_V1001, } from '../generated/types' import { - ApplicationMetadata, - IAddUpcomingOpening, - IOpeningMetadata, - IRemoveUpcomingOpening, - ISetGroupMetadata, - IWorkingGroupMetadata, - IWorkingGroupMetadataAction, - OpeningMetadata, - RemarkMetadataAction, - WorkingGroupMetadataAction, -} from '@joystream/metadata-protobuf' -import { Bytes } from '@polkadot/types' -import { - deserializeMetadata, + INT32MAX, + WorkingGroupModuleName, bytesToString, + deserializeMetadata, genericEventFields, getWorker, - WorkingGroupModuleName, - toNumber, - INT32MAX, - inconsistentState, getWorkingGroupByName, getWorkingGroupLead, + inconsistentState, invalidMetadata, + toNumber, } from './common' -import BN from 'bn.js' -import { - WorkingGroupOpening, - OpeningAddedEvent, - WorkingGroup, - WorkingGroupOpeningMetadata, - ApplicationFormQuestion, - ApplicationFormQuestionType, - OpeningStatusOpen, - WorkingGroupOpeningType, - WorkingGroupApplication, - ApplicationFormQuestionAnswer, - AppliedOnOpeningEvent, - Membership, - ApplicationStatusPending, - ApplicationStatusAccepted, - ApplicationStatusRejected, - Worker, - WorkerStatusActive, - OpeningFilledEvent, - OpeningStatusFilled, - // LeaderSetEvent, - OpeningCanceledEvent, - OpeningStatusCancelled, - ApplicationStatusCancelled, - ApplicationWithdrawnEvent, - ApplicationStatusWithdrawn, - UpcomingWorkingGroupOpening, - StatusTextChangedEvent, - WorkingGroupMetadata, - WorkingGroupMetadataSet, - UpcomingOpeningRemoved, - InvalidActionMetadata, - WorkingGroupMetadataActionResult, - UpcomingOpeningAdded, - WorkerRoleAccountUpdatedEvent, - WorkerRewardAccountUpdatedEvent, - StakeIncreasedEvent, - RewardPaidEvent, - RewardPaymentType, - NewMissedRewardLevelReachedEvent, - WorkerExitedEvent, - WorkerStatusLeft, - WorkerStatusTerminated, - TerminatedWorkerEvent, - LeaderUnsetEvent, - TerminatedLeaderEvent, - WorkerRewardAmountUpdatedEvent, - StakeSlashedEvent, - StakeDecreasedEvent, - WorkerStartedLeavingEvent, - BudgetSetEvent, - BudgetSpendingEvent, - LeaderSetEvent, - WorkerStatusLeaving, - BudgetFundedEvent, -} from 'query-node/dist/model' -import { createType } from '@joystream/types' -import { DecodedMetadataObject } from '@joystream/metadata-protobuf/types' -import { isSet } from '@joystream/metadata-protobuf/utils' import { moderatePost } from './forum' +import { processSetNodeOperationalStatusMessage } from './storage' // Reusable functions async function getWorkingGroup( @@ -714,10 +716,10 @@ export async function workingGroups_StatusTextChanged({ store, event }: EventCon } export async function workingGroups_LeadRemarked({ store, event }: EventContext & StoreContext): Promise { - const [metadataByte] = new WorkingGroup_LeadRemarkedEvent_V1001(event).params + const [metadataBytes] = new WorkingGroup_LeadRemarkedEvent_V1001(event).params const group = await getWorkingGroup(store, event) - const metadata = deserializeMetadata(RemarkMetadataAction, metadataByte) + const metadata = deserializeMetadata(LeadRemarked, metadataBytes) if (metadata?.moderatePost) { if (group.name !== 'forumWorkingGroup') { return invalidMetadata(`The ${group.name} is incompatible with the remarked moderatePost`) @@ -726,6 +728,8 @@ export async function workingGroups_LeadRemarked({ store, event }: EventContext const actor = await getWorkingGroupLead(store, group.name) await moderatePost(store, event, 'leadRemark', postId, actor, rationale) + } else if (metadata?.setNodeOperationalStatus) { + await processSetNodeOperationalStatusMessage(store, event, group, metadata.setNodeOperationalStatus) } else { return invalidMetadata('Unrecognized remarked action') } From ca8a495a95c97f982d4346687ef13576f3ee743f Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 10:53:52 +0500 Subject: [PATCH 04/18] add CLI commands to set operational status in Argus --- distributor-node/package.json | 1 + .../src/@types/inquirer-datepicker/index.d.ts | 1 + distributor-node/src/command-base/default.ts | 27 +++++-- .../leader/set-node-operational-status.ts | 81 +++++++++++++++++++ .../src/commands/operator/set-metadata.ts | 64 +++++++++++++-- .../src/schemas/operatorMetadataSchema.ts | 10 +++ distributor-node/src/types/metadata.ts | 5 ++ yarn.lock | 11 +++ 8 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 distributor-node/src/@types/inquirer-datepicker/index.d.ts create mode 100644 distributor-node/src/commands/leader/set-node-operational-status.ts create mode 100644 distributor-node/src/types/metadata.ts diff --git a/distributor-node/package.json b/distributor-node/package.json index a3bb9ca6a5..a966d25f77 100644 --- a/distributor-node/package.json +++ b/distributor-node/package.json @@ -40,6 +40,7 @@ "graphql": "^14.7.0", "graphql-tag": "^2.12.6", "inquirer": "^8.1.2", + "inquirer-datepicker": "2.0.2", "js-image-generator": "^1.0.3", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.21", diff --git a/distributor-node/src/@types/inquirer-datepicker/index.d.ts b/distributor-node/src/@types/inquirer-datepicker/index.d.ts new file mode 100644 index 0000000000..5f9d649d02 --- /dev/null +++ b/distributor-node/src/@types/inquirer-datepicker/index.d.ts @@ -0,0 +1 @@ +declare module 'inquirer-datepicker' diff --git a/distributor-node/src/command-base/default.ts b/distributor-node/src/command-base/default.ts index 1e4fecd04f..26d98ee885 100644 --- a/distributor-node/src/command-base/default.ts +++ b/distributor-node/src/command-base/default.ts @@ -1,12 +1,13 @@ import Command, { flags as oclifFlags } from '@oclif/command' -import inquirer from 'inquirer' -import ExitCodes from './ExitCodes' -import { ReadonlyConfig } from '../types/config' -import { ConfigParserService } from '../services/parsers/ConfigParserService' -import { LoggingService } from '../services/logging' +import inquirer, { DistinctQuestion } from 'inquirer' +import inquirerDatepicker from 'inquirer-datepicker' import { Logger } from 'winston' +import { LoggingService } from '../services/logging' import { BagIdParserService } from '../services/parsers/BagIdParserService' import { BucketIdParserService } from '../services/parsers/BucketIdParserService' +import { ConfigParserService } from '../services/parsers/ConfigParserService' +import { ReadonlyConfig } from '../types/config' +import ExitCodes from './ExitCodes' export const flags = { ...oclifFlags, @@ -75,6 +76,7 @@ export default abstract class DefaultCommandBase extends Command { this.logging = LoggingService.withCLIConfig() this.logger = this.logging.createLogger('CLI') this.autoConfirm = !!(process.env.AUTO_CONFIRM === 'true' || parseInt(process.env.AUTO_CONFIRM || '') || yes) + inquirer.registerPrompt('datepicker', inquirerDatepicker) } public log(message: string, ...meta: unknown[]): void { @@ -98,6 +100,21 @@ export default abstract class DefaultCommandBase extends Command { } } + async datePrompt(question: DistinctQuestion): Promise { + const { result } = await inquirer.prompt([ + { + ...question, + type: 'datepicker', + name: 'result', + clearable: true, + default: new Date('2017-09-28 17:36:05').toISOString(), + }, + ]) + + const date = new Date(result) + return date + } + async finally(err: unknown): Promise { if (!err) this.exit(ExitCodes.OK) if (process.env.DEBUG === 'true') { diff --git a/distributor-node/src/commands/leader/set-node-operational-status.ts b/distributor-node/src/commands/leader/set-node-operational-status.ts new file mode 100644 index 0000000000..14cdad9913 --- /dev/null +++ b/distributor-node/src/commands/leader/set-node-operational-status.ts @@ -0,0 +1,81 @@ +import { + INodeOperationalStatusMetadata, + ISetNodeOperationalStatus, + NodeOperationalStatusMetadata, + SetNodeOperationalStatus, +} from '@joystream/metadata-protobuf' +import AccountsCommandBase from '../../command-base/accounts' +import DefaultCommandBase, { flags } from '../../command-base/default' +import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../types/metadata' + +export default class LeadSetNodeOperationalStatus extends AccountsCommandBase { + static description = `Set/update distribution node operational status. Requires distribution working group leader permissions.` + + static flags = { + bucketId: flags.bucketId({ + required: true, + }), + workerId: flags.integer({ + char: 'w', + description: 'ID of the operator (distribution group worker)', + required: true, + }), + operationalStatus: flags.enum({ + char: 'o', + options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + required: false, + description: 'Operational status of the operator', + }), + ...DefaultCommandBase.flags, + } + + async run(): Promise { + const { bucketId, workerId, operationalStatus: statusType } = this.parse(LeadSetNodeOperationalStatus).flags + const leadKey = await this.getDistributorLeadKey() + + let operationalStatus: INodeOperationalStatusMetadata = {} + switch (statusType) { + case 'Normal': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + break + } + case 'NoService': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + break + } + case 'NoServiceFrom': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + } + break + } + case 'NoServiceDuring': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + } + } + } + + this.log(`Setting node operational status...`, { + bucketId: bucketId.toHuman(), + workerId, + operationalStatus, + }) + + const metadata: ISetNodeOperationalStatus = { + workerId: workerId.toString(), + bucketId: `${bucketId.distributionBucketFamilyId}:${bucketId.distributionBucketIndex}`, + operationalStatus, + } + await this.sendAndFollowTx( + await this.getDecodedPair(leadKey), + this.api.tx.distributionWorkingGroup.leadRemark( + '0x' + Buffer.from(SetNodeOperationalStatus.encode(metadata).finish()).toString('hex') + ) + ) + this.log('Bucket operator metadata successfully set/updated!') + } +} diff --git a/distributor-node/src/commands/operator/set-metadata.ts b/distributor-node/src/commands/operator/set-metadata.ts index e8fae81f3c..763f166af9 100644 --- a/distributor-node/src/commands/operator/set-metadata.ts +++ b/distributor-node/src/commands/operator/set-metadata.ts @@ -1,8 +1,14 @@ +import { + DistributionBucketOperatorMetadata, + IDistributionBucketOperatorMetadata, + INodeOperationalStatusMetadata, + NodeOperationalStatusMetadata, +} from '@joystream/metadata-protobuf' import fs from 'fs' import AccountsCommandBase from '../../command-base/accounts' import DefaultCommandBase, { flags } from '../../command-base/default' import { ValidationService } from '../../services/validation/ValidationService' -import { DistributionBucketOperatorMetadata, IDistributionBucketOperatorMetadata } from '@joystream/metadata-protobuf' +import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../types/metadata' export default class OperatorSetMetadata extends AccountsCommandBase { static description = `Set/update distribution bucket operator metadata. @@ -22,6 +28,12 @@ export default class OperatorSetMetadata extends AccountsCommandBase { description: 'Root distribution node endpoint', exclusive: ['input'], }), + operationalStatus: flags.enum({ + char: 'o', + options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + required: false, + description: 'Operational status of the operator', + }), input: flags.string({ char: 'i', description: 'Path to JSON metadata file', @@ -31,18 +43,58 @@ export default class OperatorSetMetadata extends AccountsCommandBase { } async run(): Promise { - const { bucketId, workerId, input, endpoint } = this.parse(OperatorSetMetadata).flags + const { bucketId, workerId, input, endpoint, operationalStatus: statusType } = this.parse(OperatorSetMetadata).flags const workerKey = await this.getDistributorWorkerRoleKey(workerId) + let operationalStatus: INodeOperationalStatusMetadata = {} + switch (statusType) { + case 'Normal': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + break + } + case 'NoService': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + break + } + case 'NoServiceFrom': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + } + break + } + case 'NoServiceDuring': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + } + } + } + const validation = new ValidationService() - const metadata: IDistributionBucketOperatorMetadata = input - ? validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) - : { endpoint } + let metadata: IDistributionBucketOperatorMetadata + if (input) { + const params = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) + metadata = { + ...params, + operationalStatus: { + ...params.operationalStatus, + status: + params.operationalStatus?.status === 'Normal' + ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL + : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + }, + } + } else { + metadata = { endpoint, operationalStatus } + } this.log(`Setting bucket operator metadata...`, { bucketId: bucketId.toHuman(), workerId, metadata, + operationalStatus, }) await this.sendAndFollowTx( await this.getDecodedPair(workerKey), @@ -52,6 +104,6 @@ export default class OperatorSetMetadata extends AccountsCommandBase { '0x' + Buffer.from(DistributionBucketOperatorMetadata.encode(metadata).finish()).toString('hex') ) ) - this.log('Bucket operator metadata succesfully set/updated!') + this.log('Bucket operator metadata successfully set/updated!') } } diff --git a/distributor-node/src/schemas/operatorMetadataSchema.ts b/distributor-node/src/schemas/operatorMetadataSchema.ts index 9bf2fb4d9d..d9c2d759da 100644 --- a/distributor-node/src/schemas/operatorMetadataSchema.ts +++ b/distributor-node/src/schemas/operatorMetadataSchema.ts @@ -1,4 +1,5 @@ import { JSONSchema4 } from 'json-schema' +import { NODE_OPERATIONAL_STATUS_OPTIONS } from '../types/metadata' export const operatorMetadataSchema: JSONSchema4 = { type: 'object', @@ -21,6 +22,15 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, }, + operationalStatus: { + type: 'object', + properties: { + status: { type: 'string', enum: [...NODE_OPERATIONAL_STATUS_OPTIONS] }, + noServiceFrom: { type: 'string', format: 'date-time' }, + noServiceTo: { type: 'string', format: 'date-time' }, + rationale: { type: 'string' }, + }, + }, extra: { type: 'string' }, }, } diff --git a/distributor-node/src/types/metadata.ts b/distributor-node/src/types/metadata.ts new file mode 100644 index 0000000000..ad57d1c299 --- /dev/null +++ b/distributor-node/src/types/metadata.ts @@ -0,0 +1,5 @@ +// Distributor Node's operational states (are part of Operator metadata) +export const NODE_OPERATIONAL_STATUS_OPTIONS = ['Normal', 'NoService', 'NoServiceFrom', 'NoServiceDuring'] as const + +// convert NODE_OPERATIONAL_STATUS_OPTIONS into string literal union type +export type NodeOperationalStatus = typeof NODE_OPERATIONAL_STATUS_OPTIONS[number] diff --git a/yarn.lock b/yarn.lock index 687c9cad41..acc3ef89b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11794,6 +11794,17 @@ inquirer-datepicker-prompt@^0.4.2: lodash "^4.17.4" util "^0.10.3" +inquirer-datepicker@2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/inquirer-datepicker/-/inquirer-datepicker-2.0.2.tgz#eaf331a9489585b336c5bd42598c25eda32ba377" + integrity sha512-Jyb8nc383c/q2hwhTXhUyjAWV2GXHahlET/Fn/WtJWX4+05z8i4YUnNHj50f6W/5oTPy4G2/NlYNlEcZmcJC/Q== + dependencies: + chalk "^4.1.2" + cli-cursor "^3.1.0" + lodash "^4.17.21" + moment "^2.29.3" + rxjs "^7.5.5" + inquirer@8.2.2: version "8.2.2" resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-8.2.2.tgz#1310517a87a0814d25336c78a20b44c3d9b7629d" From e53f69601028be08f3f587b5f19ae73254b16e74 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 10:54:37 +0500 Subject: [PATCH 05/18] add CLI commands to set operational status in Colossus --- storage-node/package.json | 1 + .../src/@types/inquirer-datepicker/index.d.ts | 1 + .../src/command-base/ApiCommandBase.ts | 33 +++++-- .../leader/set-node-operational-status.ts | 88 +++++++++++++++++++ .../src/commands/operator/set-metadata.ts | 67 ++++++++++++-- .../schemas/operatorMetadataSchema.ts | 10 +++ .../src/services/metadata/schemas/types.ts | 8 +- .../src/services/runtime/extrinsics.ts | 47 +++++++++- 8 files changed, 235 insertions(+), 20 deletions(-) create mode 100644 storage-node/src/@types/inquirer-datepicker/index.d.ts create mode 100644 storage-node/src/commands/leader/set-node-operational-status.ts diff --git a/storage-node/package.json b/storage-node/package.json index 781b40d1df..ee6a1bc631 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -41,6 +41,7 @@ "fast-folder-size": "^1.4.0", "fast-safe-stringify": "^2.1.1", "file-type": "^16.5.0", + "inquirer": "^8.1.2", "lodash": "^4.17.21", "multihashes": "^4.0.2", "node-cache": "^5.1.2", diff --git a/storage-node/src/@types/inquirer-datepicker/index.d.ts b/storage-node/src/@types/inquirer-datepicker/index.d.ts new file mode 100644 index 0000000000..5f9d649d02 --- /dev/null +++ b/storage-node/src/@types/inquirer-datepicker/index.d.ts @@ -0,0 +1 @@ +declare module 'inquirer-datepicker' diff --git a/storage-node/src/command-base/ApiCommandBase.ts b/storage-node/src/command-base/ApiCommandBase.ts index 2c2a95d81d..0ce10a0a01 100644 --- a/storage-node/src/command-base/ApiCommandBase.ts +++ b/storage-node/src/command-base/ApiCommandBase.ts @@ -1,14 +1,16 @@ import { Command, flags } from '@oclif/command' -import { createApi } from '../services/runtime/api' -import { getAccountFromJsonFile, getAlicePair, getAccountFromUri } from '../services/runtime/accounts' -import { parseBagId } from '../services/helpers/bagTypes' -import { KeyringPair } from '@polkadot/keyring/types' -import { ApiPromise } from '@polkadot/api' -import logger from '../services/logger' -import ExitCodes from './ExitCodes' import { CLIError } from '@oclif/errors' import { Input } from '@oclif/parser' +import { ApiPromise } from '@polkadot/api' +import { KeyringPair } from '@polkadot/keyring/types' +import inquirer, { DistinctQuestion } from 'inquirer' +import inquirerDatepicker from 'inquirer-datepicker' import _ from 'lodash' +import { parseBagId } from '../services/helpers/bagTypes' +import logger from '../services/logger' +import { getAccountFromJsonFile, getAccountFromUri, getAlicePair } from '../services/runtime/accounts' +import { createApi } from '../services/runtime/api' +import ExitCodes from './ExitCodes' /** * Parent class for all runtime-based commands. Defines common functions. @@ -103,6 +105,8 @@ export default abstract class ApiCommandBase extends Command { } await this.getApi() + + inquirer.registerPrompt('datepicker', inquirerDatepicker) } /** @@ -178,6 +182,21 @@ export default abstract class ApiCommandBase extends Command { } } + async datePrompt(question: DistinctQuestion): Promise { + const { result } = await inquirer.prompt([ + { + ...question, + type: 'datepicker', + name: 'result', + clearable: true, + default: new Date('2017-09-28 17:36:05').toISOString(), + }, + ]) + + const date = new Date(result) + return date + } + /** * Helper-function for exit after the CLI command. It changes the exit code * depending on the previous extrinsic call success. diff --git a/storage-node/src/commands/leader/set-node-operational-status.ts b/storage-node/src/commands/leader/set-node-operational-status.ts new file mode 100644 index 0000000000..5069924d66 --- /dev/null +++ b/storage-node/src/commands/leader/set-node-operational-status.ts @@ -0,0 +1,88 @@ +import { INodeOperationalStatusMetadata, NodeOperationalStatusMetadata } from '@joystream/metadata-protobuf' +import { flags } from '@oclif/command' +import ApiCommandBase from '../../command-base/ApiCommandBase' +import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../services/metadata/schemas' +import { setStorageNodeOperationalStatus } from '../../services/runtime/extrinsics' + +/** + * CLI command: + * et/update storage node operational status. + * + * @remarks + * Storage working group leader command. Requires storage WG leader privileges. + * Shell command: "leader:set-node-operational-status" + */ +export default class LeadSetNodeOperationalStatus extends ApiCommandBase { + static description = `Set/update storage node operational status. Requires storage working group leader permissions.` + + static flags = { + bucketId: flags.integer({ + char: 'i', + required: true, + description: 'Storage bucket ID', + }), + workerId: flags.integer({ + char: 'w', + description: 'ID of the operator (distribution group worker)', + required: true, + }), + operationalStatus: flags.enum({ + char: 'o', + options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + required: false, + description: 'Operational status of the operator', + }), + ...ApiCommandBase.flags, + } + + async run(): Promise { + const { flags } = this.parse(LeadSetNodeOperationalStatus) + + const account = this.getAccount(flags) + const api = await this.getApi() + + let operationalStatus: INodeOperationalStatusMetadata = {} + switch (flags.operationalStatus) { + case 'Normal': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + break + } + case 'NoService': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + break + } + case 'NoServiceFrom': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + } + break + } + case 'NoServiceDuring': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + } + } + } + + this.log(`Setting node operational status...`, { + bucketId: flags.bucketId, + workerId: flags.workerId, + operationalStatus, + }) + + const success = await setStorageNodeOperationalStatus( + api, + account, + flags.workerId, + flags.bucketId, + operationalStatus + ) + + this.exitAfterRuntimeCall(success) + + this.log('Successfully set storage node operational status!') + } +} diff --git a/storage-node/src/commands/operator/set-metadata.ts b/storage-node/src/commands/operator/set-metadata.ts index 9d1035a336..e60d23b18e 100644 --- a/storage-node/src/commands/operator/set-metadata.ts +++ b/storage-node/src/commands/operator/set-metadata.ts @@ -1,10 +1,16 @@ +import { + INodeOperationalStatusMetadata, + IStorageBucketOperatorMetadata, + NodeOperationalStatusMetadata, + StorageBucketOperatorMetadata, +} from '@joystream/metadata-protobuf' import { flags } from '@oclif/command' -import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' +import fs from 'fs' import ApiCommandBase from '../../command-base/ApiCommandBase' import logger from '../../services/logger' +import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../services/metadata/schemas' import { ValidationService } from '../../services/metadata/validationService' -import { StorageBucketOperatorMetadata, IStorageBucketOperatorMetadata } from '@joystream/metadata-protobuf' -import fs from 'fs' +import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' /** * CLI command: * Sets metadata for the storage bucket. @@ -33,6 +39,12 @@ export default class OperatorSetMetadata extends ApiCommandBase { description: 'Root distribution node endpoint', exclusive: ['jsonFile'], }), + operationalStatus: flags.enum({ + char: 'o', + options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + required: false, + description: 'Operational status of the operator', + }), jsonFile: flags.string({ char: 'j', description: 'Path to JSON metadata file', @@ -43,16 +55,55 @@ export default class OperatorSetMetadata extends ApiCommandBase { async run(): Promise { const { flags } = this.parse(OperatorSetMetadata) - const { operatorId, bucketId, jsonFile, endpoint } = flags + const { operatorId, bucketId, jsonFile, endpoint, operationalStatus: statusType } = flags + + let operationalStatus: INodeOperationalStatusMetadata = {} + switch (statusType) { + case 'Normal': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + break + } + case 'NoService': { + operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + break + } + case 'NoServiceFrom': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + } + break + } + case 'NoServiceDuring': { + operationalStatus = { + status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + } + } + } const validation = new ValidationService() - const metadata: IStorageBucketOperatorMetadata = jsonFile - ? validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) - : { endpoint } + let metadata: IStorageBucketOperatorMetadata + if (jsonFile) { + const input = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) + metadata = { + ...input, + operationalStatus: { + ...input.operationalStatus, + status: + input.operationalStatus?.status === 'Normal' + ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL + : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + }, + } + } else { + metadata = { endpoint, operationalStatus } + } const encodedMetadata = '0x' + Buffer.from(StorageBucketOperatorMetadata.encode(metadata).finish()).toString('hex') - logger.info('Setting the storage operator metadata...') + logger.info('Se tting the storage node operational status...') if (flags.dev) { await this.ensureDevelopmentChain() } diff --git a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts index 1e6356647f..fba41c78d8 100644 --- a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts +++ b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts @@ -1,4 +1,5 @@ import { JSONSchema4 } from 'json-schema' +import { NODE_OPERATIONAL_STATUS_OPTIONS } from './types' // Storage node operator metadata JSON schema. export const operatorMetadataSchema: JSONSchema4 = { @@ -22,6 +23,15 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, }, + operationalStatus: { + type: 'object', + properties: { + status: { type: 'string', enum: [...NODE_OPERATIONAL_STATUS_OPTIONS] }, + noServiceFrom: { type: 'string', format: 'date-time' }, + noServiceTo: { type: 'string', format: 'date-time' }, + rationale: { type: 'string' }, + }, + }, extra: { type: 'string' }, }, } diff --git a/storage-node/src/services/metadata/schemas/types.ts b/storage-node/src/services/metadata/schemas/types.ts index f8ae49d234..ad49629ef4 100644 --- a/storage-node/src/services/metadata/schemas/types.ts +++ b/storage-node/src/services/metadata/schemas/types.ts @@ -1,6 +1,12 @@ -import { schemas } from './schemas' import { OperatorMetadataJson } from '../generated/OperatorMetadataJson' +import { schemas } from './schemas' export type SchemaKey = keyof typeof schemas & string export type TypeBySchemaKey = T extends 'OperatorMetadata' ? OperatorMetadataJson : never + +// Distributor Node's operational states (are part of Operator metadata) +export const NODE_OPERATIONAL_STATUS_OPTIONS = ['Normal', 'NoService', 'NoServiceFrom', 'NoServiceDuring'] as const + +// convert NODE_OPERATIONAL_STATUS_OPTIONS into string literal union type +export type NodeOperationalStatus = typeof NODE_OPERATIONAL_STATUS_OPTIONS[number] diff --git a/storage-node/src/services/runtime/extrinsics.ts b/storage-node/src/services/runtime/extrinsics.ts index 1adfe68a84..4e25bc34e1 100644 --- a/storage-node/src/services/runtime/extrinsics.ts +++ b/storage-node/src/services/runtime/extrinsics.ts @@ -1,10 +1,15 @@ -import { sendAndFollowNamedTx, getEvent } from './api' -import { KeyringPair } from '@polkadot/keyring/types' +import { + INodeOperationalStatusMetadata, + ISetNodeOperationalStatus, + SetNodeOperationalStatus, +} from '@joystream/metadata-protobuf' import { ApiPromise } from '@polkadot/api' +import { KeyringPair } from '@polkadot/keyring/types' import { PalletStorageBagIdType as BagId, PalletStorageDynamicBagType as DynamicBagType } from '@polkadot/types/lookup' -import logger from '../../services/logger' -import { timeout } from 'promise-timeout' import BN from 'bn.js' +import { timeout } from 'promise-timeout' +import logger from '../../services/logger' +import { getEvent, sendAndFollowNamedTx } from './api' /** * Creates storage bucket. @@ -494,3 +499,37 @@ export async function updateBlacklist( return sendAndFollowNamedTx(api, account, tx) }) } + +/** + * Set/update storage nodes operational status by Lead + * + * @remarks + * It sends an lead remark extrinsic to the runtime. + * + * @param api - runtime API promise + * @param account - KeyringPair instance + * @param workerId - Worker ID + * @param buckerId - Bucket ID + * @param operationalStatus - Operational Status to set for the node + * @returns promise with a success flag. + */ +export async function setStorageNodeOperationalStatus( + api: ApiPromise, + account: KeyringPair, + workerId: number, + bucketId: number, + operationalStatus: INodeOperationalStatusMetadata +): Promise { + return await extrinsicWrapper(() => { + const metadata: ISetNodeOperationalStatus = { + workerId: workerId.toString(), + bucketId: bucketId.toString(), + operationalStatus, + } + const tx = api.tx.storageWorkingGroup.leadRemark( + '0x' + Buffer.from(SetNodeOperationalStatus.encode(metadata).finish()).toString('hex') + ) + + return sendAndFollowNamedTx(api, account, tx) + }) +} From 21bc8077dfdf10336982dc1dc8ac9350cc07078a Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 21 Jun 2023 11:06:21 +0500 Subject: [PATCH 06/18] argus: generate-docs --- distributor-node/docs/commands/leader.md | 28 +++++++++++++++++++ distributor-node/docs/commands/operator.md | 20 ++++++++----- ...es-endpoints-properties-joystreamnodews.md | 4 +++ ...operties-endpoints-properties-querynode.md | 4 +++ .../schema/definition-properties-endpoints.md | 8 ++++++ ...rch-logging-options-properties-endpoint.md | 4 +++ ...roperties-elasticsearch-logging-options.md | 4 +++ 7 files changed, 65 insertions(+), 7 deletions(-) diff --git a/distributor-node/docs/commands/leader.md b/distributor-node/docs/commands/leader.md index 9f5f6efbea..94ddf18c39 100644 --- a/distributor-node/docs/commands/leader.md +++ b/distributor-node/docs/commands/leader.md @@ -12,6 +12,7 @@ Commands for performing Distribution Working Group leader on-chain duties (like * [`joystream-distributor leader:remove-bucket-operator`](#joystream-distributor-leaderremove-bucket-operator) * [`joystream-distributor leader:set-bucket-family-metadata`](#joystream-distributor-leaderset-bucket-family-metadata) * [`joystream-distributor leader:set-buckets-per-bag-limit`](#joystream-distributor-leaderset-buckets-per-bag-limit) +* [`joystream-distributor leader:set-node-operational-status`](#joystream-distributor-leaderset-node-operational-status) * [`joystream-distributor leader:update-bag`](#joystream-distributor-leaderupdate-bag) * [`joystream-distributor leader:update-bucket-mode`](#joystream-distributor-leaderupdate-bucket-mode) * [`joystream-distributor leader:update-bucket-status`](#joystream-distributor-leaderupdate-bucket-status) @@ -210,6 +211,33 @@ OPTIONS _See code: [src/commands/leader/set-buckets-per-bag-limit.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/set-buckets-per-bag-limit.ts)_ +## `joystream-distributor leader:set-node-operational-status` + +Set/update distribution node operational status. Requires distribution working group leader permissions. + +``` +USAGE + $ joystream-distributor leader:set-node-operational-status + +OPTIONS + -B, --bucketId=bucketId (required) Distribution bucket ID in + {familyId}:{bucketIndex} format. + + -c, --configPath=configPath [default: ./config.yml] Path to config + JSON/YAML file (relative to current working + directory) + + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator + + -w, --workerId=workerId (required) ID of the operator (distribution + group worker) + + -y, --yes Answer "yes" to any prompt, skipping any + manual confirmations +``` + +_See code: [src/commands/leader/set-node-operational-status.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/set-node-operational-status.ts)_ + ## `joystream-distributor leader:update-bag` Add/remove distribution buckets from a bag. diff --git a/distributor-node/docs/commands/operator.md b/distributor-node/docs/commands/operator.md index 907978fb87..d60235d069 100644 --- a/distributor-node/docs/commands/operator.md +++ b/distributor-node/docs/commands/operator.md @@ -39,18 +39,24 @@ USAGE $ joystream-distributor operator:set-metadata OPTIONS - -B, --bucketId=bucketId (required) Distribution bucket ID in {familyId}:{bucketIndex} format. + -B, --bucketId=bucketId (required) Distribution bucket ID in + {familyId}:{bucketIndex} format. - -c, --configPath=configPath [default: ./config.yml] Path to config JSON/YAML file (relative to current working - directory) + -c, --configPath=configPath [default: ./config.yml] Path to config + JSON/YAML file (relative to current working + directory) - -e, --endpoint=endpoint Root distribution node endpoint + -e, --endpoint=endpoint Root distribution node endpoint - -i, --input=input Path to JSON metadata file + -i, --input=input Path to JSON metadata file - -w, --workerId=workerId (required) ID of the operator (distribution group worker) + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator - -y, --yes Answer "yes" to any prompt, skipping any manual confirmations + -w, --workerId=workerId (required) ID of the operator (distribution + group worker) + + -y, --yes Answer "yes" to any prompt, skipping any + manual confirmations DESCRIPTION Requires active distribution bucket operator worker role key. diff --git a/distributor-node/docs/schema/definition-properties-endpoints-properties-joystreamnodews.md b/distributor-node/docs/schema/definition-properties-endpoints-properties-joystreamnodews.md index aba750d91b..8b646d9ef5 100644 --- a/distributor-node/docs/schema/definition-properties-endpoints-properties-joystreamnodews.md +++ b/distributor-node/docs/schema/definition-properties-endpoints-properties-joystreamnodews.md @@ -1,3 +1,7 @@ ## joystreamNodeWs Type `string` + +## joystreamNodeWs Constraints + +**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification") diff --git a/distributor-node/docs/schema/definition-properties-endpoints-properties-querynode.md b/distributor-node/docs/schema/definition-properties-endpoints-properties-querynode.md index 6dd1888bdc..b47f837206 100644 --- a/distributor-node/docs/schema/definition-properties-endpoints-properties-querynode.md +++ b/distributor-node/docs/schema/definition-properties-endpoints-properties-querynode.md @@ -1,3 +1,7 @@ ## queryNode Type `string` + +## queryNode Constraints + +**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification") diff --git a/distributor-node/docs/schema/definition-properties-endpoints.md b/distributor-node/docs/schema/definition-properties-endpoints.md index 370fc862c1..c11ae19007 100644 --- a/distributor-node/docs/schema/definition-properties-endpoints.md +++ b/distributor-node/docs/schema/definition-properties-endpoints.md @@ -27,6 +27,10 @@ Query node graphql server uri (for example: ) `string` +### queryNode Constraints + +**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification") + ## joystreamNodeWs Joystream node websocket api uri (for example: ws\://localhost:9944) @@ -44,3 +48,7 @@ Joystream node websocket api uri (for example: ws\://localhost:9944) ### joystreamNodeWs Type `string` + +### joystreamNodeWs Constraints + +**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification") diff --git a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md index 00e8b7f7be..39367d55d5 100644 --- a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md +++ b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md @@ -1,3 +1,7 @@ ## endpoint Type `string` + +## endpoint Constraints + +**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification") diff --git a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md index 0e5b8939d4..4cd4948674 100644 --- a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md +++ b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md @@ -61,6 +61,10 @@ Elastichsearch endpoint to push the logs to (for example: Date: Tue, 18 Jul 2023 07:00:43 +0000 Subject: [PATCH 07/18] add inquirer dependency in storage-node package.json --- .../networking/query-node/schema.graphql | 342 ++++++++++++++++++ storage-node/package.json | 2 + .../src/services/queryNode/schema.graphql | 342 ++++++++++++++++++ yarn.lock | 5 + 4 files changed, 691 insertions(+) diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql index dfa50ea0d9..ddde3f91a2 100644 --- a/distributor-node/src/services/networking/query-node/schema.graphql +++ b/distributor-node/src/services/networking/query-node/schema.graphql @@ -89,6 +89,7 @@ enum EventTypeOptions { CommentTextUpdatedEvent CouncilBudgetFundedEvent CouncilorRewardUpdatedEvent + DistributionNodeOperationalStatusSetEvent EnglishAuctionSettledEvent EnglishAuctionStartedEvent InitialInvitationBalanceUpdatedEvent @@ -154,6 +155,7 @@ enum EventTypeOptions { StakingAccountConfirmedEvent StakingAccountRemovedEvent StatusTextChangedEvent + StorageNodeOperationalStatusSetEvent TerminatedLeaderEvent TerminatedWorkerEvent ThreadCreatedEvent @@ -916,6 +918,8 @@ type DistributionBucket implements BaseGraphQLObject { bags: [StorageBag!]! } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring + type DistributionBucketOperator implements BaseGraphQLObject { id: ID! createdAt: DateTime! @@ -935,6 +939,7 @@ type DistributionBucketOperator implements BaseGraphQLObject { status: DistributionBucketOperatorStatus! metadata: DistributionBucketOperatorMetadata metadataId: String + distributionnodeoperationalstatusseteventbucketOperator: [DistributionNodeOperationalStatusSetEvent!] } enum DistributionBucketOperatorStatus { @@ -957,6 +962,9 @@ type DistributionBucketOperatorMetadata implements BaseGraphQLObject { nodeLocation: NodeLocationMetadata nodeLocationId: String + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String distributionbucketoperatormetadata: [DistributionBucketOperator!] @@ -998,6 +1006,9 @@ type StorageBucketOperatorMetadata implements BaseGraphQLObject { nodeLocation: NodeLocationMetadata nodeLocationId: String + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String storagebucketoperatorMetadata: [StorageBucket!] @@ -1033,6 +1044,7 @@ type StorageBucket implements BaseGraphQLObject { """Total size of assigned data objects""" dataObjectsSize: BigInt! + storagenodeoperationalstatusseteventstorageBucket: [StorageNodeOperationalStatusSetEvent!] } union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive @@ -2506,6 +2518,50 @@ type MetaprotocolTransactionSuccessful { channelPaid: Channel } +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceDuring { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + to: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + type OpeningStatusCancelled { """Related event emitted on opening cancellation""" openingCanceledEvent: OpeningCanceledEvent @@ -4439,6 +4495,17 @@ type DistributionBucketConnection { pageInfo: PageInfo! } +type DistributionNodeOperationalStatusSetEventEdge { + node: DistributionNodeOperationalStatusSetEvent! + cursor: String! +} + +type DistributionNodeOperationalStatusSetEventConnection { + totalCount: Int! + edges: [DistributionNodeOperationalStatusSetEventEdge!]! + pageInfo: PageInfo! +} + type ElectedCouncilEdge { node: ElectedCouncil! cursor: String! @@ -5525,6 +5592,17 @@ type StorageDataObjectConnection { pageInfo: PageInfo! } +type StorageNodeOperationalStatusSetEventEdge { + node: StorageNodeOperationalStatusSetEvent! + cursor: String! +} + +type StorageNodeOperationalStatusSetEventConnection { + totalCount: Int! + edges: [StorageNodeOperationalStatusSetEventEdge!]! + pageInfo: PageInfo! +} + type TerminatedLeaderEventEdge { node: TerminatedLeaderEvent! cursor: String! @@ -12123,6 +12201,79 @@ input DistributionBucketUpdateInput { distributing: Boolean } +input DistributionNodeOperationalStatusSetEventWhereInput { + id_eq: ID + id_in: [ID!] + createdAt_eq: DateTime + createdAt_lt: DateTime + createdAt_lte: DateTime + createdAt_gt: DateTime + createdAt_gte: DateTime + createdById_eq: ID + createdById_in: [ID!] + updatedAt_eq: DateTime + updatedAt_lt: DateTime + updatedAt_lte: DateTime + updatedAt_gt: DateTime + updatedAt_gte: DateTime + updatedById_eq: ID + updatedById_in: [ID!] + deletedAt_all: Boolean + deletedAt_eq: DateTime + deletedAt_lt: DateTime + deletedAt_lte: DateTime + deletedAt_gt: DateTime + deletedAt_gte: DateTime + deletedById_eq: ID + deletedById_in: [ID!] + inExtrinsic_eq: String + inExtrinsic_contains: String + inExtrinsic_startsWith: String + inExtrinsic_endsWith: String + inExtrinsic_in: [String!] + inBlock_eq: Int + inBlock_gt: Int + inBlock_gte: Int + inBlock_lt: Int + inBlock_lte: Int + inBlock_in: [Int!] + network_eq: Network + network_in: [Network!] + indexInBlock_eq: Int + indexInBlock_gt: Int + indexInBlock_gte: Int + indexInBlock_lt: Int + indexInBlock_lte: Int + indexInBlock_in: [Int!] + operationalStatus_json: JSONObject + bucketOperator: DistributionBucketOperatorWhereInput + AND: [DistributionNodeOperationalStatusSetEventWhereInput!] + OR: [DistributionNodeOperationalStatusSetEventWhereInput!] + NOT: [DistributionNodeOperationalStatusSetEventWhereInput!] +} + +input DistributionNodeOperationalStatusSetEventWhereUniqueInput { + id: ID! +} + +input DistributionNodeOperationalStatusSetEventCreateInput { + inExtrinsic: String + inBlock: Float! + network: Network! + indexInBlock: Float! + bucketOperator: ID + operationalStatus: JSONObject! +} + +input DistributionNodeOperationalStatusSetEventUpdateInput { + inExtrinsic: String + inBlock: Float + network: Network + indexInBlock: Float + bucketOperator: ID + operationalStatus: JSONObject +} + input DistributionBucketOperatorWhereInput { id_eq: ID id_in: [ID!] @@ -12158,6 +12309,9 @@ input DistributionBucketOperatorWhereInput { status_in: [DistributionBucketOperatorStatus!] distributionBucket: DistributionBucketWhereInput metadata: DistributionBucketOperatorMetadataWhereInput + distributionnodeoperationalstatusseteventbucketOperator_none: DistributionNodeOperationalStatusSetEventWhereInput + distributionnodeoperationalstatusseteventbucketOperator_some: DistributionNodeOperationalStatusSetEventWhereInput + distributionnodeoperationalstatusseteventbucketOperator_every: DistributionNodeOperationalStatusSetEventWhereInput AND: [DistributionBucketOperatorWhereInput!] OR: [DistributionBucketOperatorWhereInput!] NOT: [DistributionBucketOperatorWhereInput!] @@ -12211,6 +12365,7 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_startsWith: String nodeEndpoint_endsWith: String nodeEndpoint_in: [String!] + nodeOperationalStatus_json: JSONObject extra_eq: String extra_contains: String extra_startsWith: String @@ -12232,12 +12387,14 @@ input DistributionBucketOperatorMetadataWhereUniqueInput { input DistributionBucketOperatorMetadataCreateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject! extra: String } input DistributionBucketOperatorMetadataUpdateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject extra: String } @@ -12334,6 +12491,7 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_startsWith: String nodeEndpoint_endsWith: String nodeEndpoint_in: [String!] + nodeOperationalStatus_json: JSONObject extra_eq: String extra_contains: String extra_startsWith: String @@ -12355,15 +12513,90 @@ input StorageBucketOperatorMetadataWhereUniqueInput { input StorageBucketOperatorMetadataCreateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject! extra: String } input StorageBucketOperatorMetadataUpdateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject extra: String } +input StorageNodeOperationalStatusSetEventWhereInput { + id_eq: ID + id_in: [ID!] + createdAt_eq: DateTime + createdAt_lt: DateTime + createdAt_lte: DateTime + createdAt_gt: DateTime + createdAt_gte: DateTime + createdById_eq: ID + createdById_in: [ID!] + updatedAt_eq: DateTime + updatedAt_lt: DateTime + updatedAt_lte: DateTime + updatedAt_gt: DateTime + updatedAt_gte: DateTime + updatedById_eq: ID + updatedById_in: [ID!] + deletedAt_all: Boolean + deletedAt_eq: DateTime + deletedAt_lt: DateTime + deletedAt_lte: DateTime + deletedAt_gt: DateTime + deletedAt_gte: DateTime + deletedById_eq: ID + deletedById_in: [ID!] + inExtrinsic_eq: String + inExtrinsic_contains: String + inExtrinsic_startsWith: String + inExtrinsic_endsWith: String + inExtrinsic_in: [String!] + inBlock_eq: Int + inBlock_gt: Int + inBlock_gte: Int + inBlock_lt: Int + inBlock_lte: Int + inBlock_in: [Int!] + network_eq: Network + network_in: [Network!] + indexInBlock_eq: Int + indexInBlock_gt: Int + indexInBlock_gte: Int + indexInBlock_lt: Int + indexInBlock_lte: Int + indexInBlock_in: [Int!] + operationalStatus_json: JSONObject + storageBucket: StorageBucketWhereInput + AND: [StorageNodeOperationalStatusSetEventWhereInput!] + OR: [StorageNodeOperationalStatusSetEventWhereInput!] + NOT: [StorageNodeOperationalStatusSetEventWhereInput!] +} + +input StorageNodeOperationalStatusSetEventWhereUniqueInput { + id: ID! +} + +input StorageNodeOperationalStatusSetEventCreateInput { + inExtrinsic: String + inBlock: Float! + network: Network! + indexInBlock: Float! + storageBucket: ID + operationalStatus: JSONObject! +} + +input StorageNodeOperationalStatusSetEventUpdateInput { + inExtrinsic: String + inBlock: Float + network: Network + indexInBlock: Float + storageBucket: ID + operationalStatus: JSONObject +} + input StorageBucketWhereInput { id_eq: ID id_in: [ID!] @@ -12420,6 +12653,9 @@ input StorageBucketWhereInput { bags_none: StorageBagWhereInput bags_some: StorageBagWhereInput bags_every: StorageBagWhereInput + storagenodeoperationalstatusseteventstorageBucket_none: StorageNodeOperationalStatusSetEventWhereInput + storagenodeoperationalstatusseteventstorageBucket_some: StorageNodeOperationalStatusSetEventWhereInput + storagenodeoperationalstatusseteventstorageBucket_every: StorageNodeOperationalStatusSetEventWhereInput AND: [StorageBucketWhereInput!] OR: [StorageBucketWhereInput!] NOT: [StorageBucketWhereInput!] @@ -25102,6 +25338,68 @@ type OpeningFilledEvent implements Event & BaseGraphQLObject { workersHired: [Worker!]! } +type DistributionNodeOperationalStatusSetEvent implements Event & BaseGraphQLObject { + id: ID! + createdAt: DateTime! + createdById: ID! + updatedAt: DateTime + updatedById: ID + deletedAt: DateTime + deletedById: ID + version: Int! + + """Hash of the extrinsic which caused the event to be emitted""" + inExtrinsic: String + + """Blocknumber of the block in which the event was emitted.""" + inBlock: Int! + + """Network the block was produced in""" + network: Network! + + """Index of event in block from which it was emitted.""" + indexInBlock: Int! + + """Filtering options for interface implementers""" + type: EventTypeOptions + bucketOperator: DistributionBucketOperator + bucketOperatorId: String + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + +type StorageNodeOperationalStatusSetEvent implements Event & BaseGraphQLObject { + id: ID! + createdAt: DateTime! + createdById: ID! + updatedAt: DateTime + updatedById: ID + deletedAt: DateTime + deletedById: ID + version: Int! + + """Hash of the extrinsic which caused the event to be emitted""" + inExtrinsic: String + + """Blocknumber of the block in which the event was emitted.""" + inBlock: Int! + + """Network the block was produced in""" + network: Network! + + """Index of event in block from which it was emitted.""" + indexInBlock: Int! + + """Filtering options for interface implementers""" + type: EventTypeOptions + storageBucket: StorageBucket + storageBucketId: String + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type AuctionBidCanceledEvent implements Event & BaseGraphQLObject { id: ID! createdAt: DateTime! @@ -27892,6 +28190,9 @@ type Query { distributionBuckets(offset: Int, limit: Int = 50, where: DistributionBucketWhereInput, orderBy: [DistributionBucketOrderByInput!]): [DistributionBucket!]! distributionBucketByUniqueInput(where: DistributionBucketWhereUniqueInput!): DistributionBucket distributionBucketsConnection(first: Int, after: String, last: Int, before: String, where: DistributionBucketWhereInput, orderBy: [DistributionBucketOrderByInput!]): DistributionBucketConnection! + distributionNodeOperationalStatusSetEvents(offset: Int, limit: Int = 50, where: DistributionNodeOperationalStatusSetEventWhereInput, orderBy: [DistributionNodeOperationalStatusSetEventOrderByInput!]): [DistributionNodeOperationalStatusSetEvent!]! + distributionNodeOperationalStatusSetEventByUniqueInput(where: DistributionNodeOperationalStatusSetEventWhereUniqueInput!): DistributionNodeOperationalStatusSetEvent + distributionNodeOperationalStatusSetEventsConnection(first: Int, after: String, last: Int, before: String, where: DistributionNodeOperationalStatusSetEventWhereInput, orderBy: [DistributionNodeOperationalStatusSetEventOrderByInput!]): DistributionNodeOperationalStatusSetEventConnection! electedCouncils(offset: Int, limit: Int = 50, where: ElectedCouncilWhereInput, orderBy: [ElectedCouncilOrderByInput!]): [ElectedCouncil!]! electedCouncilByUniqueInput(where: ElectedCouncilWhereUniqueInput!): ElectedCouncil electedCouncilsConnection(first: Int, after: String, last: Int, before: String, where: ElectedCouncilWhereInput, orderBy: [ElectedCouncilOrderByInput!]): ElectedCouncilConnection! @@ -28179,6 +28480,9 @@ type Query { storageDataObjects(offset: Int, limit: Int = 50, where: StorageDataObjectWhereInput, orderBy: [StorageDataObjectOrderByInput!]): [StorageDataObject!]! storageDataObjectByUniqueInput(where: StorageDataObjectWhereUniqueInput!): StorageDataObject storageDataObjectsConnection(first: Int, after: String, last: Int, before: String, where: StorageDataObjectWhereInput, orderBy: [StorageDataObjectOrderByInput!]): StorageDataObjectConnection! + storageNodeOperationalStatusSetEvents(offset: Int, limit: Int = 50, where: StorageNodeOperationalStatusSetEventWhereInput, orderBy: [StorageNodeOperationalStatusSetEventOrderByInput!]): [StorageNodeOperationalStatusSetEvent!]! + storageNodeOperationalStatusSetEventByUniqueInput(where: StorageNodeOperationalStatusSetEventWhereUniqueInput!): StorageNodeOperationalStatusSetEvent + storageNodeOperationalStatusSetEventsConnection(first: Int, after: String, last: Int, before: String, where: StorageNodeOperationalStatusSetEventWhereInput, orderBy: [StorageNodeOperationalStatusSetEventOrderByInput!]): StorageNodeOperationalStatusSetEventConnection! terminatedLeaderEvents(offset: Int, limit: Int = 50, where: TerminatedLeaderEventWhereInput, orderBy: [TerminatedLeaderEventOrderByInput!]): [TerminatedLeaderEvent!]! terminatedLeaderEventByUniqueInput(where: TerminatedLeaderEventWhereUniqueInput!): TerminatedLeaderEvent terminatedLeaderEventsConnection(first: Int, after: String, last: Int, before: String, where: TerminatedLeaderEventWhereInput, orderBy: [TerminatedLeaderEventOrderByInput!]): TerminatedLeaderEventConnection! @@ -29958,6 +30262,25 @@ enum DistributionBucketOrderByInput { distributing_DESC } +enum DistributionNodeOperationalStatusSetEventOrderByInput { + createdAt_ASC + createdAt_DESC + updatedAt_ASC + updatedAt_DESC + deletedAt_ASC + deletedAt_DESC + inExtrinsic_ASC + inExtrinsic_DESC + inBlock_ASC + inBlock_DESC + network_ASC + network_DESC + indexInBlock_ASC + indexInBlock_DESC + bucketOperator_ASC + bucketOperator_DESC +} + enum ElectedCouncilOrderByInput { createdAt_ASC createdAt_DESC @@ -31914,6 +32237,25 @@ enum StorageDataObjectOrderByInput { unsetAt_DESC } +enum StorageNodeOperationalStatusSetEventOrderByInput { + createdAt_ASC + createdAt_DESC + updatedAt_ASC + updatedAt_DESC + deletedAt_ASC + deletedAt_DESC + inExtrinsic_ASC + inExtrinsic_DESC + inBlock_ASC + inBlock_DESC + network_ASC + network_DESC + indexInBlock_ASC + indexInBlock_DESC + storageBucket_ASC + storageBucket_DESC +} + enum TerminatedLeaderEventOrderByInput { createdAt_ASC createdAt_DESC diff --git a/storage-node/package.json b/storage-node/package.json index 3698bc0ccc..4fceaa4c9a 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -20,6 +20,7 @@ "@types/base64url": "^2.0.0", "@types/express": "4.17.13", "@types/file-type": "^10.9.1", + "@types/inquirer": "^6.5.0", "@types/lodash": "^4.14.171", "@types/mkdirp": "^0.5.1", "@types/multer": "^1.4.5", @@ -45,6 +46,7 @@ "fast-safe-stringify": "^2.1.1", "file-type": "^16.5.0", "inquirer": "^8.1.2", + "inquirer-datepicker": "2.0.2", "lodash": "^4.17.21", "mkdirp": "^0.5.1", "multihashes": "^4.0.2", diff --git a/storage-node/src/services/queryNode/schema.graphql b/storage-node/src/services/queryNode/schema.graphql index dfa50ea0d9..ddde3f91a2 100644 --- a/storage-node/src/services/queryNode/schema.graphql +++ b/storage-node/src/services/queryNode/schema.graphql @@ -89,6 +89,7 @@ enum EventTypeOptions { CommentTextUpdatedEvent CouncilBudgetFundedEvent CouncilorRewardUpdatedEvent + DistributionNodeOperationalStatusSetEvent EnglishAuctionSettledEvent EnglishAuctionStartedEvent InitialInvitationBalanceUpdatedEvent @@ -154,6 +155,7 @@ enum EventTypeOptions { StakingAccountConfirmedEvent StakingAccountRemovedEvent StatusTextChangedEvent + StorageNodeOperationalStatusSetEvent TerminatedLeaderEvent TerminatedWorkerEvent ThreadCreatedEvent @@ -916,6 +918,8 @@ type DistributionBucket implements BaseGraphQLObject { bags: [StorageBag!]! } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring + type DistributionBucketOperator implements BaseGraphQLObject { id: ID! createdAt: DateTime! @@ -935,6 +939,7 @@ type DistributionBucketOperator implements BaseGraphQLObject { status: DistributionBucketOperatorStatus! metadata: DistributionBucketOperatorMetadata metadataId: String + distributionnodeoperationalstatusseteventbucketOperator: [DistributionNodeOperationalStatusSetEvent!] } enum DistributionBucketOperatorStatus { @@ -957,6 +962,9 @@ type DistributionBucketOperatorMetadata implements BaseGraphQLObject { nodeLocation: NodeLocationMetadata nodeLocationId: String + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String distributionbucketoperatormetadata: [DistributionBucketOperator!] @@ -998,6 +1006,9 @@ type StorageBucketOperatorMetadata implements BaseGraphQLObject { nodeLocation: NodeLocationMetadata nodeLocationId: String + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String storagebucketoperatorMetadata: [StorageBucket!] @@ -1033,6 +1044,7 @@ type StorageBucket implements BaseGraphQLObject { """Total size of assigned data objects""" dataObjectsSize: BigInt! + storagenodeoperationalstatusseteventstorageBucket: [StorageNodeOperationalStatusSetEvent!] } union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive @@ -2506,6 +2518,50 @@ type MetaprotocolTransactionSuccessful { channelPaid: Channel } +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceDuring { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + to: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + type OpeningStatusCancelled { """Related event emitted on opening cancellation""" openingCanceledEvent: OpeningCanceledEvent @@ -4439,6 +4495,17 @@ type DistributionBucketConnection { pageInfo: PageInfo! } +type DistributionNodeOperationalStatusSetEventEdge { + node: DistributionNodeOperationalStatusSetEvent! + cursor: String! +} + +type DistributionNodeOperationalStatusSetEventConnection { + totalCount: Int! + edges: [DistributionNodeOperationalStatusSetEventEdge!]! + pageInfo: PageInfo! +} + type ElectedCouncilEdge { node: ElectedCouncil! cursor: String! @@ -5525,6 +5592,17 @@ type StorageDataObjectConnection { pageInfo: PageInfo! } +type StorageNodeOperationalStatusSetEventEdge { + node: StorageNodeOperationalStatusSetEvent! + cursor: String! +} + +type StorageNodeOperationalStatusSetEventConnection { + totalCount: Int! + edges: [StorageNodeOperationalStatusSetEventEdge!]! + pageInfo: PageInfo! +} + type TerminatedLeaderEventEdge { node: TerminatedLeaderEvent! cursor: String! @@ -12123,6 +12201,79 @@ input DistributionBucketUpdateInput { distributing: Boolean } +input DistributionNodeOperationalStatusSetEventWhereInput { + id_eq: ID + id_in: [ID!] + createdAt_eq: DateTime + createdAt_lt: DateTime + createdAt_lte: DateTime + createdAt_gt: DateTime + createdAt_gte: DateTime + createdById_eq: ID + createdById_in: [ID!] + updatedAt_eq: DateTime + updatedAt_lt: DateTime + updatedAt_lte: DateTime + updatedAt_gt: DateTime + updatedAt_gte: DateTime + updatedById_eq: ID + updatedById_in: [ID!] + deletedAt_all: Boolean + deletedAt_eq: DateTime + deletedAt_lt: DateTime + deletedAt_lte: DateTime + deletedAt_gt: DateTime + deletedAt_gte: DateTime + deletedById_eq: ID + deletedById_in: [ID!] + inExtrinsic_eq: String + inExtrinsic_contains: String + inExtrinsic_startsWith: String + inExtrinsic_endsWith: String + inExtrinsic_in: [String!] + inBlock_eq: Int + inBlock_gt: Int + inBlock_gte: Int + inBlock_lt: Int + inBlock_lte: Int + inBlock_in: [Int!] + network_eq: Network + network_in: [Network!] + indexInBlock_eq: Int + indexInBlock_gt: Int + indexInBlock_gte: Int + indexInBlock_lt: Int + indexInBlock_lte: Int + indexInBlock_in: [Int!] + operationalStatus_json: JSONObject + bucketOperator: DistributionBucketOperatorWhereInput + AND: [DistributionNodeOperationalStatusSetEventWhereInput!] + OR: [DistributionNodeOperationalStatusSetEventWhereInput!] + NOT: [DistributionNodeOperationalStatusSetEventWhereInput!] +} + +input DistributionNodeOperationalStatusSetEventWhereUniqueInput { + id: ID! +} + +input DistributionNodeOperationalStatusSetEventCreateInput { + inExtrinsic: String + inBlock: Float! + network: Network! + indexInBlock: Float! + bucketOperator: ID + operationalStatus: JSONObject! +} + +input DistributionNodeOperationalStatusSetEventUpdateInput { + inExtrinsic: String + inBlock: Float + network: Network + indexInBlock: Float + bucketOperator: ID + operationalStatus: JSONObject +} + input DistributionBucketOperatorWhereInput { id_eq: ID id_in: [ID!] @@ -12158,6 +12309,9 @@ input DistributionBucketOperatorWhereInput { status_in: [DistributionBucketOperatorStatus!] distributionBucket: DistributionBucketWhereInput metadata: DistributionBucketOperatorMetadataWhereInput + distributionnodeoperationalstatusseteventbucketOperator_none: DistributionNodeOperationalStatusSetEventWhereInput + distributionnodeoperationalstatusseteventbucketOperator_some: DistributionNodeOperationalStatusSetEventWhereInput + distributionnodeoperationalstatusseteventbucketOperator_every: DistributionNodeOperationalStatusSetEventWhereInput AND: [DistributionBucketOperatorWhereInput!] OR: [DistributionBucketOperatorWhereInput!] NOT: [DistributionBucketOperatorWhereInput!] @@ -12211,6 +12365,7 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_startsWith: String nodeEndpoint_endsWith: String nodeEndpoint_in: [String!] + nodeOperationalStatus_json: JSONObject extra_eq: String extra_contains: String extra_startsWith: String @@ -12232,12 +12387,14 @@ input DistributionBucketOperatorMetadataWhereUniqueInput { input DistributionBucketOperatorMetadataCreateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject! extra: String } input DistributionBucketOperatorMetadataUpdateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject extra: String } @@ -12334,6 +12491,7 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_startsWith: String nodeEndpoint_endsWith: String nodeEndpoint_in: [String!] + nodeOperationalStatus_json: JSONObject extra_eq: String extra_contains: String extra_startsWith: String @@ -12355,15 +12513,90 @@ input StorageBucketOperatorMetadataWhereUniqueInput { input StorageBucketOperatorMetadataCreateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject! extra: String } input StorageBucketOperatorMetadataUpdateInput { nodeEndpoint: String nodeLocation: ID + nodeOperationalStatus: JSONObject extra: String } +input StorageNodeOperationalStatusSetEventWhereInput { + id_eq: ID + id_in: [ID!] + createdAt_eq: DateTime + createdAt_lt: DateTime + createdAt_lte: DateTime + createdAt_gt: DateTime + createdAt_gte: DateTime + createdById_eq: ID + createdById_in: [ID!] + updatedAt_eq: DateTime + updatedAt_lt: DateTime + updatedAt_lte: DateTime + updatedAt_gt: DateTime + updatedAt_gte: DateTime + updatedById_eq: ID + updatedById_in: [ID!] + deletedAt_all: Boolean + deletedAt_eq: DateTime + deletedAt_lt: DateTime + deletedAt_lte: DateTime + deletedAt_gt: DateTime + deletedAt_gte: DateTime + deletedById_eq: ID + deletedById_in: [ID!] + inExtrinsic_eq: String + inExtrinsic_contains: String + inExtrinsic_startsWith: String + inExtrinsic_endsWith: String + inExtrinsic_in: [String!] + inBlock_eq: Int + inBlock_gt: Int + inBlock_gte: Int + inBlock_lt: Int + inBlock_lte: Int + inBlock_in: [Int!] + network_eq: Network + network_in: [Network!] + indexInBlock_eq: Int + indexInBlock_gt: Int + indexInBlock_gte: Int + indexInBlock_lt: Int + indexInBlock_lte: Int + indexInBlock_in: [Int!] + operationalStatus_json: JSONObject + storageBucket: StorageBucketWhereInput + AND: [StorageNodeOperationalStatusSetEventWhereInput!] + OR: [StorageNodeOperationalStatusSetEventWhereInput!] + NOT: [StorageNodeOperationalStatusSetEventWhereInput!] +} + +input StorageNodeOperationalStatusSetEventWhereUniqueInput { + id: ID! +} + +input StorageNodeOperationalStatusSetEventCreateInput { + inExtrinsic: String + inBlock: Float! + network: Network! + indexInBlock: Float! + storageBucket: ID + operationalStatus: JSONObject! +} + +input StorageNodeOperationalStatusSetEventUpdateInput { + inExtrinsic: String + inBlock: Float + network: Network + indexInBlock: Float + storageBucket: ID + operationalStatus: JSONObject +} + input StorageBucketWhereInput { id_eq: ID id_in: [ID!] @@ -12420,6 +12653,9 @@ input StorageBucketWhereInput { bags_none: StorageBagWhereInput bags_some: StorageBagWhereInput bags_every: StorageBagWhereInput + storagenodeoperationalstatusseteventstorageBucket_none: StorageNodeOperationalStatusSetEventWhereInput + storagenodeoperationalstatusseteventstorageBucket_some: StorageNodeOperationalStatusSetEventWhereInput + storagenodeoperationalstatusseteventstorageBucket_every: StorageNodeOperationalStatusSetEventWhereInput AND: [StorageBucketWhereInput!] OR: [StorageBucketWhereInput!] NOT: [StorageBucketWhereInput!] @@ -25102,6 +25338,68 @@ type OpeningFilledEvent implements Event & BaseGraphQLObject { workersHired: [Worker!]! } +type DistributionNodeOperationalStatusSetEvent implements Event & BaseGraphQLObject { + id: ID! + createdAt: DateTime! + createdById: ID! + updatedAt: DateTime + updatedById: ID + deletedAt: DateTime + deletedById: ID + version: Int! + + """Hash of the extrinsic which caused the event to be emitted""" + inExtrinsic: String + + """Blocknumber of the block in which the event was emitted.""" + inBlock: Int! + + """Network the block was produced in""" + network: Network! + + """Index of event in block from which it was emitted.""" + indexInBlock: Int! + + """Filtering options for interface implementers""" + type: EventTypeOptions + bucketOperator: DistributionBucketOperator + bucketOperatorId: String + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + +type StorageNodeOperationalStatusSetEvent implements Event & BaseGraphQLObject { + id: ID! + createdAt: DateTime! + createdById: ID! + updatedAt: DateTime + updatedById: ID + deletedAt: DateTime + deletedById: ID + version: Int! + + """Hash of the extrinsic which caused the event to be emitted""" + inExtrinsic: String + + """Blocknumber of the block in which the event was emitted.""" + inBlock: Int! + + """Network the block was produced in""" + network: Network! + + """Index of event in block from which it was emitted.""" + indexInBlock: Int! + + """Filtering options for interface implementers""" + type: EventTypeOptions + storageBucket: StorageBucket + storageBucketId: String + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type AuctionBidCanceledEvent implements Event & BaseGraphQLObject { id: ID! createdAt: DateTime! @@ -27892,6 +28190,9 @@ type Query { distributionBuckets(offset: Int, limit: Int = 50, where: DistributionBucketWhereInput, orderBy: [DistributionBucketOrderByInput!]): [DistributionBucket!]! distributionBucketByUniqueInput(where: DistributionBucketWhereUniqueInput!): DistributionBucket distributionBucketsConnection(first: Int, after: String, last: Int, before: String, where: DistributionBucketWhereInput, orderBy: [DistributionBucketOrderByInput!]): DistributionBucketConnection! + distributionNodeOperationalStatusSetEvents(offset: Int, limit: Int = 50, where: DistributionNodeOperationalStatusSetEventWhereInput, orderBy: [DistributionNodeOperationalStatusSetEventOrderByInput!]): [DistributionNodeOperationalStatusSetEvent!]! + distributionNodeOperationalStatusSetEventByUniqueInput(where: DistributionNodeOperationalStatusSetEventWhereUniqueInput!): DistributionNodeOperationalStatusSetEvent + distributionNodeOperationalStatusSetEventsConnection(first: Int, after: String, last: Int, before: String, where: DistributionNodeOperationalStatusSetEventWhereInput, orderBy: [DistributionNodeOperationalStatusSetEventOrderByInput!]): DistributionNodeOperationalStatusSetEventConnection! electedCouncils(offset: Int, limit: Int = 50, where: ElectedCouncilWhereInput, orderBy: [ElectedCouncilOrderByInput!]): [ElectedCouncil!]! electedCouncilByUniqueInput(where: ElectedCouncilWhereUniqueInput!): ElectedCouncil electedCouncilsConnection(first: Int, after: String, last: Int, before: String, where: ElectedCouncilWhereInput, orderBy: [ElectedCouncilOrderByInput!]): ElectedCouncilConnection! @@ -28179,6 +28480,9 @@ type Query { storageDataObjects(offset: Int, limit: Int = 50, where: StorageDataObjectWhereInput, orderBy: [StorageDataObjectOrderByInput!]): [StorageDataObject!]! storageDataObjectByUniqueInput(where: StorageDataObjectWhereUniqueInput!): StorageDataObject storageDataObjectsConnection(first: Int, after: String, last: Int, before: String, where: StorageDataObjectWhereInput, orderBy: [StorageDataObjectOrderByInput!]): StorageDataObjectConnection! + storageNodeOperationalStatusSetEvents(offset: Int, limit: Int = 50, where: StorageNodeOperationalStatusSetEventWhereInput, orderBy: [StorageNodeOperationalStatusSetEventOrderByInput!]): [StorageNodeOperationalStatusSetEvent!]! + storageNodeOperationalStatusSetEventByUniqueInput(where: StorageNodeOperationalStatusSetEventWhereUniqueInput!): StorageNodeOperationalStatusSetEvent + storageNodeOperationalStatusSetEventsConnection(first: Int, after: String, last: Int, before: String, where: StorageNodeOperationalStatusSetEventWhereInput, orderBy: [StorageNodeOperationalStatusSetEventOrderByInput!]): StorageNodeOperationalStatusSetEventConnection! terminatedLeaderEvents(offset: Int, limit: Int = 50, where: TerminatedLeaderEventWhereInput, orderBy: [TerminatedLeaderEventOrderByInput!]): [TerminatedLeaderEvent!]! terminatedLeaderEventByUniqueInput(where: TerminatedLeaderEventWhereUniqueInput!): TerminatedLeaderEvent terminatedLeaderEventsConnection(first: Int, after: String, last: Int, before: String, where: TerminatedLeaderEventWhereInput, orderBy: [TerminatedLeaderEventOrderByInput!]): TerminatedLeaderEventConnection! @@ -29958,6 +30262,25 @@ enum DistributionBucketOrderByInput { distributing_DESC } +enum DistributionNodeOperationalStatusSetEventOrderByInput { + createdAt_ASC + createdAt_DESC + updatedAt_ASC + updatedAt_DESC + deletedAt_ASC + deletedAt_DESC + inExtrinsic_ASC + inExtrinsic_DESC + inBlock_ASC + inBlock_DESC + network_ASC + network_DESC + indexInBlock_ASC + indexInBlock_DESC + bucketOperator_ASC + bucketOperator_DESC +} + enum ElectedCouncilOrderByInput { createdAt_ASC createdAt_DESC @@ -31914,6 +32237,25 @@ enum StorageDataObjectOrderByInput { unsetAt_DESC } +enum StorageNodeOperationalStatusSetEventOrderByInput { + createdAt_ASC + createdAt_DESC + updatedAt_ASC + updatedAt_DESC + deletedAt_ASC + deletedAt_DESC + inExtrinsic_ASC + inExtrinsic_DESC + inBlock_ASC + inBlock_DESC + network_ASC + network_DESC + indexInBlock_ASC + indexInBlock_DESC + storageBucket_ASC + storageBucket_DESC +} + enum TerminatedLeaderEventOrderByInput { createdAt_ASC createdAt_DESC diff --git a/yarn.lock b/yarn.lock index 88763065ec..a53f4cb904 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15182,6 +15182,11 @@ moment@^2.22.1, moment@^2.24.0, moment@^2.29.1: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== +moment@^2.29.3: + version "2.29.4" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== + monitor-event-loop-delay@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/monitor-event-loop-delay/-/monitor-event-loop-delay-1.0.0.tgz#b5ab78165a3bb93f2b275c50d01430c7f155d1f7" From d133ffbf1f3265f560d5aaece632820c08c21e0f Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Tue, 1 Aug 2023 15:18:17 +0000 Subject: [PATCH 08/18] bump package versions & update changelog --- distributor-node/CHANGELOG.md | 5 +++++ distributor-node/package.json | 4 ++-- joystreamjs/package.json | 2 +- metadata-protobuf/package.json | 2 +- metadata-protobuf/proto/Storage.proto | 1 - query-node/CHANGELOG.md | 4 ++++ storage-node/CHANGELOG.md | 5 +++++ storage-node/package.json | 4 ++-- 8 files changed, 20 insertions(+), 7 deletions(-) diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md index 2a5a428282..cc75d7b4b6 100644 --- a/distributor-node/CHANGELOG.md +++ b/distributor-node/CHANGELOG.md @@ -1,3 +1,8 @@ +### 1.3.0 + +- Updates `operator:set-metadata` CLI command to set distributor-node's operational status along with other metadata. +- Adds `leader:set-node-operational-status` CLI command to set operational status of any distributor-node by Lead. + ### 1.2.2 - **FIX** `sendExtrinsic`: The send extrinsic function (which is a wrapper around PolkadotJS `tx.signAndSend` function) has been fixed to handle the case when tx has been finalized before the callback registered in `tx.signAndSend` would run. diff --git a/distributor-node/package.json b/distributor-node/package.json index 4b143a96b8..549cb8db9d 100644 --- a/distributor-node/package.json +++ b/distributor-node/package.json @@ -1,7 +1,7 @@ { "name": "@joystream/distributor-cli", "description": "Joystream distributor node CLI", - "version": "1.2.2", + "version": "1.3.0", "author": "Joystream contributors", "bin": { "joystream-distributor": "./bin/run" @@ -10,7 +10,7 @@ "dependencies": { "@apollo/client": "^3.2.5", "@elastic/ecs-winston-format": "^1.1.0", - "@joystream/metadata-protobuf": "^2.8.0", + "@joystream/metadata-protobuf": "^2.9.0", "@joystream/opentelemetry": "1.0.0", "@joystream/storage-node-client": "^3.0.0", "@joystream/types": "^2.0.0", diff --git a/joystreamjs/package.json b/joystreamjs/package.json index 45d0307cb0..45504f27e8 100644 --- a/joystreamjs/package.json +++ b/joystreamjs/package.json @@ -41,7 +41,7 @@ "generate:all": "yarn generate:schema-typings" }, "dependencies": { - "@joystream/metadata-protobuf": "^2.8.1", + "@joystream/metadata-protobuf": "^2.9.0", "@joystream/types": "^2.0.0", "@polkadot/util-crypto": "9.5.1", "axios": "^1.2.1", diff --git a/metadata-protobuf/package.json b/metadata-protobuf/package.json index 90c614ef74..0692180b56 100644 --- a/metadata-protobuf/package.json +++ b/metadata-protobuf/package.json @@ -1,6 +1,6 @@ { "name": "@joystream/metadata-protobuf", - "version": "2.8.1", + "version": "2.9.0", "description": "Joystream Metadata Protobuf Library", "main": "lib/index.js", "types": "lib/index.d.ts", diff --git a/metadata-protobuf/proto/Storage.proto b/metadata-protobuf/proto/Storage.proto index 93bece2cbd..8313dfa25a 100644 --- a/metadata-protobuf/proto/Storage.proto +++ b/metadata-protobuf/proto/Storage.proto @@ -16,7 +16,6 @@ message StorageBucketOperatorMetadata { optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator optional NodeOperationalStatusMetadata operational_status = 4; // Node's operational status to set - } message DistributionBucketOperatorMetadata { diff --git a/query-node/CHANGELOG.md b/query-node/CHANGELOG.md index 05a4efd9d6..c84fd500c2 100644 --- a/query-node/CHANGELOG.md +++ b/query-node/CHANGELOG.md @@ -1,3 +1,7 @@ +### 1.3.0 + +- Updates mappings to process `NodeOperationalStatusMetadata` protobuf message. This metaprotocol message enables both storage/distribution workers & leads to set the operational status of the nodes. + ### 1.2.2 - Integrates OpenTelemetry API/SDK with Query Node's Graphql Server for exporting improved tracing logs & metrics to Elasticsearch. Adds `./start-elasticsearch-stack.sh` script to bootstrap elasticsearch services (Elasticsearch + Kibana + APM Server) with all the required configurations. diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md index e1ba1a017a..8138e2095f 100644 --- a/storage-node/CHANGELOG.md +++ b/storage-node/CHANGELOG.md @@ -1,3 +1,8 @@ +### 3.8.0 + +- Updates `operator:set-metadata` CLI command to set storage-node's operational status along with other metadata. +- Adds `leader:set-node-operational-status` CLI command to set operational status of any storage-node by Lead. + ### 3.7.0 - Updates `leader:update-bag` CLI command to `leader:update-bags` to accept multiple bag ids as input. This allows the command to be used to update storage buckets of multiple bags in a single batched transaction. diff --git a/storage-node/package.json b/storage-node/package.json index 4790b7a64e..d05c43ed00 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -1,7 +1,7 @@ { "name": "storage-node", "description": "Joystream storage subsystem.", - "version": "3.7.0", + "version": "3.8.0", "author": "Joystream contributors", "bin": { "storage-node": "./bin/run" @@ -10,7 +10,7 @@ "dependencies": { "@apollo/client": "^3.3.21", "@elastic/ecs-winston-format": "^1.3.1", - "@joystream/metadata-protobuf": "^2.8.0", + "@joystream/metadata-protobuf": "^2.9.0", "@joystream/opentelemetry": "1.0.0", "@joystream/types": "^2.0.0", "@oclif/command": "^1", From ba24cfe683ee6c29f3f5ba43c9152d4736a4f50a Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 2 Aug 2023 13:40:59 +0000 Subject: [PATCH 09/18] bump QN package version --- query-node/mappings/package.json | 2 +- query-node/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/query-node/mappings/package.json b/query-node/mappings/package.json index 6393616252..ace55598ed 100644 --- a/query-node/mappings/package.json +++ b/query-node/mappings/package.json @@ -1,6 +1,6 @@ { "name": "query-node-mappings", - "version": "1.2.1", + "version": "1.3.0", "description": "Mappings for hydra-processor", "main": "lib/src/index.js", "license": "MIT", diff --git a/query-node/package.json b/query-node/package.json index f6a4fb6575..67130bf6a1 100644 --- a/query-node/package.json +++ b/query-node/package.json @@ -1,6 +1,6 @@ { "name": "query-node-root", - "version": "1.2.2", + "version": "1.3.0", "description": "GraphQL server and mappings. Generated with ♥ by Hydra-CLI", "scripts": { "build": "./build.sh", From 6ff45319432ff711ce35e281f01b0fba889089a3 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 11:23:34 +0500 Subject: [PATCH 10/18] address requested changes --- .../src/commands/operator/set-metadata.ts | 16 +++++++++------- .../leader/set-node-operational-status.ts | 2 +- .../src/commands/operator/set-metadata.ts | 18 +++++++++--------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/distributor-node/src/commands/operator/set-metadata.ts b/distributor-node/src/commands/operator/set-metadata.ts index 763f166af9..62bbb383b4 100644 --- a/distributor-node/src/commands/operator/set-metadata.ts +++ b/distributor-node/src/commands/operator/set-metadata.ts @@ -78,13 +78,15 @@ export default class OperatorSetMetadata extends AccountsCommandBase { const params = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) metadata = { ...params, - operationalStatus: { - ...params.operationalStatus, - status: - params.operationalStatus?.status === 'Normal' - ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL - : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - }, + operationalStatus: params.operationalStatus + ? { + ...params.operationalStatus, + status: + params.operationalStatus?.status === 'Normal' + ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL + : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + } + : {}, } } else { metadata = { endpoint, operationalStatus } diff --git a/storage-node/src/commands/leader/set-node-operational-status.ts b/storage-node/src/commands/leader/set-node-operational-status.ts index cff00edf92..b278860947 100644 --- a/storage-node/src/commands/leader/set-node-operational-status.ts +++ b/storage-node/src/commands/leader/set-node-operational-status.ts @@ -23,7 +23,7 @@ export default class LeadSetNodeOperationalStatus extends LeaderCommandBase { }), workerId: flags.integer({ char: 'w', - description: 'ID of the operator (distribution group worker)', + description: 'ID of the operator (storage group worker)', required: true, }), operationalStatus: flags.enum({ diff --git a/storage-node/src/commands/operator/set-metadata.ts b/storage-node/src/commands/operator/set-metadata.ts index 5e2cc964cb..ee38dd48b1 100644 --- a/storage-node/src/commands/operator/set-metadata.ts +++ b/storage-node/src/commands/operator/set-metadata.ts @@ -7,7 +7,6 @@ import { import { flags } from '@oclif/command' import fs from 'fs' import ApiCommandBase from '../../command-base/ApiCommandBase' -import logger from '../../services/logger' import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../services/metadata/schemas' import { ValidationService } from '../../services/metadata/validationService' import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' @@ -91,13 +90,15 @@ export default class OperatorSetMetadata extends ApiCommandBase { const input = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) metadata = { ...input, - operationalStatus: { - ...input.operationalStatus, - status: - input.operationalStatus?.status === 'Normal' - ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL - : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - }, + operationalStatus: input.operationalStatus + ? { + ...input.operationalStatus, + status: + input.operationalStatus?.status === 'Normal' + ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL + : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, + } + : {}, } } else { metadata = { endpoint, operationalStatus } @@ -105,7 +106,6 @@ export default class OperatorSetMetadata extends ApiCommandBase { const encodedMetadata = '0x' + Buffer.from(StorageBucketOperatorMetadata.encode(metadata).finish()).toString('hex') - logger.info('Se tting the storage node operational status...') if (flags.dev) { await this.ensureDevelopmentChain() } From fc04793e3823b85ad93289e6170a89759bcbe1ff Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 11:24:26 +0500 Subject: [PATCH 11/18] update generated storage-squid schema --- .../networking/query-node/schema.graphql | 230 +++++++++++++++++- .../src/services/queryNode/schema.graphql | 230 +++++++++++++++++- 2 files changed, 458 insertions(+), 2 deletions(-) diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql index bb21100008..c0ba380b15 100644 --- a/distributor-node/src/services/networking/query-node/schema.graphql +++ b/distributor-node/src/services/networking/query-node/schema.graphql @@ -373,6 +373,9 @@ type DistributionBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -417,6 +420,26 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_to_ASC + nodeOperationalStatus_to_DESC + nodeOperationalStatus_to_ASC_NULLS_FIRST + nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -462,6 +485,8 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -651,6 +676,14 @@ input DistributionBucketWhereInput { OR: [DistributionBucketWhereInput!] } +type DistributionNodeOperationalStatusSetEvent { + """Distribution bucket operator""" + bucketOperator: DistributionBucketOperator + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type Event { """{blockNumber}-{indexInBlock}""" id: String! @@ -671,7 +704,7 @@ type Event { data: EventData! } -union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData +union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData | StorageNodeOperationalStatusSetEvent | DistributionNodeOperationalStatusSetEvent input EventDataWhereInput { result_isNull: Boolean @@ -693,6 +726,12 @@ input EventDataWhereInput { dataObjectId_not_startsWith: String dataObjectId_endsWith: String dataObjectId_not_endsWith: String + storageBucket_isNull: Boolean + storageBucket: StorageBucketWhereInput + operationalStatus_isNull: Boolean + operationalStatus: NodeOperationalStatusWhereInput + bucketOperator_isNull: Boolean + bucketOperator: DistributionBucketOperatorWhereInput isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -975,6 +1014,113 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceDuring { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + to: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +input NodeOperationalStatusWhereInput { + rationale_isNull: Boolean + rationale_eq: String + rationale_not_eq: String + rationale_gt: String + rationale_gte: String + rationale_lt: String + rationale_lte: String + rationale_in: [String!] + rationale_not_in: [String!] + rationale_contains: String + rationale_not_contains: String + rationale_containsInsensitive: String + rationale_not_containsInsensitive: String + rationale_startsWith: String + rationale_not_startsWith: String + rationale_endsWith: String + rationale_not_endsWith: String + forced_isNull: Boolean + forced_eq: Boolean + forced_not_eq: Boolean + from_isNull: Boolean + from_eq: DateTime + from_not_eq: DateTime + from_gt: DateTime + from_gte: DateTime + from_lt: DateTime + from_lte: DateTime + from_in: [DateTime!] + from_not_in: [DateTime!] + to_isNull: Boolean + to_eq: DateTime + to_not_eq: DateTime + to_gt: DateTime + to_gte: DateTime + to_lt: DateTime + to_lte: DateTime + to_in: [DateTime!] + to_not_in: [DateTime!] + isTypeOf_isNull: Boolean + isTypeOf_eq: String + isTypeOf_not_eq: String + isTypeOf_gt: String + isTypeOf_gte: String + isTypeOf_lt: String + isTypeOf_lte: String + isTypeOf_in: [String!] + isTypeOf_not_in: [String!] + isTypeOf_contains: String + isTypeOf_not_contains: String + isTypeOf_containsInsensitive: String + isTypeOf_not_containsInsensitive: String + isTypeOf_startsWith: String + isTypeOf_not_startsWith: String + isTypeOf_endsWith: String + isTypeOf_not_endsWith: String +} + type PageInfo { hasNextPage: Boolean! hasPreviousPage: Boolean! @@ -1035,6 +1181,10 @@ type Query { distributionBucketFamilyById(id: String!): DistributionBucketFamily distributionBucketFamilyByUniqueInput(where: WhereIdInput!): DistributionBucketFamily @deprecated(reason: "Use distributionBucketFamilyById") distributionBucketFamiliesConnection(orderBy: [DistributionBucketFamilyOrderByInput!]!, after: String, first: Int, where: DistributionBucketFamilyWhereInput): DistributionBucketFamiliesConnection! + workers(where: WorkerWhereInput, orderBy: [WorkerOrderByInput!], offset: Int, limit: Int): [Worker!]! + workerById(id: String!): Worker + workerByUniqueInput(where: WhereIdInput!): Worker @deprecated(reason: "Use workerById") + workersConnection(orderBy: [WorkerOrderByInput!]!, after: String, first: Int, where: WorkerWhereInput): WorkersConnection! squidStatus: SquidStatus squidVersion: SquidVersion! } @@ -1376,6 +1526,9 @@ type StorageBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -1432,6 +1585,26 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_to_ASC + nodeOperationalStatus_to_DESC + nodeOperationalStatus_to_ASC_NULLS_FIRST + nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -1477,6 +1650,8 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -1860,6 +2035,14 @@ input StorageDataObjectWhereInput { OR: [StorageDataObjectWhereInput!] } +type StorageNodeOperationalStatusSetEvent { + """Storage Bucket""" + storageBucket: StorageBucket! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type VideoSubtitle { """(videoId)-{type}-{language}""" id: String! @@ -1965,3 +2148,48 @@ input VideoSubtitleWhereInput { input WhereIdInput { id: String! } + +type Worker { + """Worker id ({workingGroupName}-{workerId})""" + id: String! +} + +type WorkerEdge { + node: Worker! + cursor: String! +} + +enum WorkerOrderByInput { + id_ASC + id_DESC + id_ASC_NULLS_FIRST + id_DESC_NULLS_LAST +} + +type WorkersConnection { + edges: [WorkerEdge!]! + pageInfo: PageInfo! + totalCount: Int! +} + +input WorkerWhereInput { + id_isNull: Boolean + id_eq: String + id_not_eq: String + id_gt: String + id_gte: String + id_lt: String + id_lte: String + id_in: [String!] + id_not_in: [String!] + id_contains: String + id_not_contains: String + id_containsInsensitive: String + id_not_containsInsensitive: String + id_startsWith: String + id_not_startsWith: String + id_endsWith: String + id_not_endsWith: String + AND: [WorkerWhereInput!] + OR: [WorkerWhereInput!] +} diff --git a/storage-node/src/services/queryNode/schema.graphql b/storage-node/src/services/queryNode/schema.graphql index bb21100008..c0ba380b15 100644 --- a/storage-node/src/services/queryNode/schema.graphql +++ b/storage-node/src/services/queryNode/schema.graphql @@ -373,6 +373,9 @@ type DistributionBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -417,6 +420,26 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_to_ASC + nodeOperationalStatus_to_DESC + nodeOperationalStatus_to_ASC_NULLS_FIRST + nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -462,6 +485,8 @@ input DistributionBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -651,6 +676,14 @@ input DistributionBucketWhereInput { OR: [DistributionBucketWhereInput!] } +type DistributionNodeOperationalStatusSetEvent { + """Distribution bucket operator""" + bucketOperator: DistributionBucketOperator + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type Event { """{blockNumber}-{indexInBlock}""" id: String! @@ -671,7 +704,7 @@ type Event { data: EventData! } -union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData +union EventData = MetaprotocolTransactionStatusEventData | DataObjectDeletedEventData | StorageNodeOperationalStatusSetEvent | DistributionNodeOperationalStatusSetEvent input EventDataWhereInput { result_isNull: Boolean @@ -693,6 +726,12 @@ input EventDataWhereInput { dataObjectId_not_startsWith: String dataObjectId_endsWith: String dataObjectId_not_endsWith: String + storageBucket_isNull: Boolean + storageBucket: StorageBucketWhereInput + operationalStatus_isNull: Boolean + operationalStatus: NodeOperationalStatusWhereInput + bucketOperator_isNull: Boolean + bucketOperator: DistributionBucketOperatorWhereInput isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -975,6 +1014,113 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring + +type NodeOperationalStatusNormal { + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoService { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceDuring { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """The time until which the bucket would have to no service""" + to: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +type NodeOperationalStatusNoServiceFrom { + """ + Whether the state was set by lead (true) or by the operator (false), it is + meant to prevent worker from unilaterally reversing. + """ + forced: Boolean! + + """The time from which the bucket would have to no service""" + from: DateTime! + + """Reason why node was set to this state""" + rationale: String +} + +input NodeOperationalStatusWhereInput { + rationale_isNull: Boolean + rationale_eq: String + rationale_not_eq: String + rationale_gt: String + rationale_gte: String + rationale_lt: String + rationale_lte: String + rationale_in: [String!] + rationale_not_in: [String!] + rationale_contains: String + rationale_not_contains: String + rationale_containsInsensitive: String + rationale_not_containsInsensitive: String + rationale_startsWith: String + rationale_not_startsWith: String + rationale_endsWith: String + rationale_not_endsWith: String + forced_isNull: Boolean + forced_eq: Boolean + forced_not_eq: Boolean + from_isNull: Boolean + from_eq: DateTime + from_not_eq: DateTime + from_gt: DateTime + from_gte: DateTime + from_lt: DateTime + from_lte: DateTime + from_in: [DateTime!] + from_not_in: [DateTime!] + to_isNull: Boolean + to_eq: DateTime + to_not_eq: DateTime + to_gt: DateTime + to_gte: DateTime + to_lt: DateTime + to_lte: DateTime + to_in: [DateTime!] + to_not_in: [DateTime!] + isTypeOf_isNull: Boolean + isTypeOf_eq: String + isTypeOf_not_eq: String + isTypeOf_gt: String + isTypeOf_gte: String + isTypeOf_lt: String + isTypeOf_lte: String + isTypeOf_in: [String!] + isTypeOf_not_in: [String!] + isTypeOf_contains: String + isTypeOf_not_contains: String + isTypeOf_containsInsensitive: String + isTypeOf_not_containsInsensitive: String + isTypeOf_startsWith: String + isTypeOf_not_startsWith: String + isTypeOf_endsWith: String + isTypeOf_not_endsWith: String +} + type PageInfo { hasNextPage: Boolean! hasPreviousPage: Boolean! @@ -1035,6 +1181,10 @@ type Query { distributionBucketFamilyById(id: String!): DistributionBucketFamily distributionBucketFamilyByUniqueInput(where: WhereIdInput!): DistributionBucketFamily @deprecated(reason: "Use distributionBucketFamilyById") distributionBucketFamiliesConnection(orderBy: [DistributionBucketFamilyOrderByInput!]!, after: String, first: Int, where: DistributionBucketFamilyWhereInput): DistributionBucketFamiliesConnection! + workers(where: WorkerWhereInput, orderBy: [WorkerOrderByInput!], offset: Int, limit: Int): [Worker!]! + workerById(id: String!): Worker + workerByUniqueInput(where: WhereIdInput!): Worker @deprecated(reason: "Use workerById") + workersConnection(orderBy: [WorkerOrderByInput!]!, after: String, first: Int, where: WorkerWhereInput): WorkersConnection! squidStatus: SquidStatus squidVersion: SquidVersion! } @@ -1376,6 +1526,9 @@ type StorageBucketOperatorMetadata { """Optional node location metadata""" nodeLocation: NodeLocationMetadata + """Optional node operational status""" + nodeOperationalStatus: NodeOperationalStatus + """Additional information about the node/operator""" extra: String } @@ -1432,6 +1585,26 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeLocation_city_DESC nodeLocation_city_ASC_NULLS_FIRST nodeLocation_city_DESC_NULLS_LAST + nodeOperationalStatus_rationale_ASC + nodeOperationalStatus_rationale_DESC + nodeOperationalStatus_rationale_ASC_NULLS_FIRST + nodeOperationalStatus_rationale_DESC_NULLS_LAST + nodeOperationalStatus_forced_ASC + nodeOperationalStatus_forced_DESC + nodeOperationalStatus_forced_ASC_NULLS_FIRST + nodeOperationalStatus_forced_DESC_NULLS_LAST + nodeOperationalStatus_from_ASC + nodeOperationalStatus_from_DESC + nodeOperationalStatus_from_ASC_NULLS_FIRST + nodeOperationalStatus_from_DESC_NULLS_LAST + nodeOperationalStatus_to_ASC + nodeOperationalStatus_to_DESC + nodeOperationalStatus_to_ASC_NULLS_FIRST + nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_isTypeOf_ASC + nodeOperationalStatus_isTypeOf_DESC + nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST + nodeOperationalStatus_isTypeOf_DESC_NULLS_LAST extra_ASC extra_DESC extra_ASC_NULLS_FIRST @@ -1477,6 +1650,8 @@ input StorageBucketOperatorMetadataWhereInput { nodeEndpoint_not_endsWith: String nodeLocation_isNull: Boolean nodeLocation: NodeLocationMetadataWhereInput + nodeOperationalStatus_isNull: Boolean + nodeOperationalStatus: NodeOperationalStatusWhereInput extra_isNull: Boolean extra_eq: String extra_not_eq: String @@ -1860,6 +2035,14 @@ input StorageDataObjectWhereInput { OR: [StorageDataObjectWhereInput!] } +type StorageNodeOperationalStatusSetEvent { + """Storage Bucket""" + storageBucket: StorageBucket! + + """Operational status that was set""" + operationalStatus: NodeOperationalStatus! +} + type VideoSubtitle { """(videoId)-{type}-{language}""" id: String! @@ -1965,3 +2148,48 @@ input VideoSubtitleWhereInput { input WhereIdInput { id: String! } + +type Worker { + """Worker id ({workingGroupName}-{workerId})""" + id: String! +} + +type WorkerEdge { + node: Worker! + cursor: String! +} + +enum WorkerOrderByInput { + id_ASC + id_DESC + id_ASC_NULLS_FIRST + id_DESC_NULLS_LAST +} + +type WorkersConnection { + edges: [WorkerEdge!]! + pageInfo: PageInfo! + totalCount: Int! +} + +input WorkerWhereInput { + id_isNull: Boolean + id_eq: String + id_not_eq: String + id_gt: String + id_gte: String + id_lt: String + id_lte: String + id_in: [String!] + id_not_in: [String!] + id_contains: String + id_not_contains: String + id_containsInsensitive: String + id_not_containsInsensitive: String + id_startsWith: String + id_not_startsWith: String + id_endsWith: String + id_not_endsWith: String + AND: [WorkerWhereInput!] + OR: [WorkerWhereInput!] +} From b95ae0413089794ce97b19f2c7f77d2791383879 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 11:25:34 +0500 Subject: [PATCH 12/18] [Argus] ensure storage node only syncs from operationally active nodes --- storage-node/src/api-spec/openapi.yaml | 16 +++++++ storage-node/src/services/queryNode/api.ts | 22 +++++++++ .../queryNode/queries/queries.graphql | 47 ++++++++++++++++++- .../services/webApi/controllers/stateApi.ts | 13 +++++ 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/storage-node/src/api-spec/openapi.yaml b/storage-node/src/api-spec/openapi.yaml index 5b86883ab8..b56b7a78c3 100644 --- a/storage-node/src/api-spec/openapi.yaml +++ b/storage-node/src/api-spec/openapi.yaml @@ -277,6 +277,7 @@ components: - downloadBuckets - sync - cleanup + - bucketsOperationalStatuses properties: version: type: string @@ -308,6 +309,21 @@ components: type: array items: type: string + bucketsOperationalStatuses: + type: array + items: + type: object + required: + - 'bucketId' + - 'status' + - 'isForced' + properties: + bucketId: + type: string + status: + type: string + isForced: + type: boolean sync: type: object required: diff --git a/storage-node/src/services/queryNode/api.ts b/storage-node/src/services/queryNode/api.ts index 44ab277d74..863474bcda 100644 --- a/storage-node/src/services/queryNode/api.ts +++ b/storage-node/src/services/queryNode/api.ts @@ -28,6 +28,9 @@ import { GetStorageBucketsByWorkerId, GetStorageBucketsByWorkerIdQuery, GetStorageBucketsByWorkerIdQueryVariables, + GetStorageBucketsOperationalStatus, + GetStorageBucketsOperationalStatusQuery, + GetStorageBucketsOperationalStatusQueryVariables, GetStorageBucketsQuery, GetStorageBucketsQueryVariables, SquidStatus, @@ -301,6 +304,25 @@ export class QueryNodeApi { return result } + /** + * Returns storage bucket IDs. + * + */ + public async getStorageBucketsOperationalStatus( + bucketIds: string[] + ): Promise> { + const result = await this.multipleEntitiesQuery< + GetStorageBucketsOperationalStatusQuery, + GetStorageBucketsOperationalStatusQueryVariables + >(GetStorageBucketsOperationalStatus, { ids: bucketIds }, 'storageBuckets') + + if (!result) { + return [] + } + + return result + } + public async getDataObjectDeletedEvents( dataObjectIds: string[] ): Promise> { diff --git a/storage-node/src/services/queryNode/queries/queries.graphql b/storage-node/src/services/queryNode/queries/queries.graphql index 8f1d1f254e..d2274e2180 100644 --- a/storage-node/src/services/queryNode/queries/queries.graphql +++ b/storage-node/src/services/queryNode/queries/queries.graphql @@ -3,19 +3,62 @@ fragment StorageBucketIds on StorageBucket { } query getStorageBuckets { - storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) { + storageBuckets( + where: { + operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } + operatorMetadata: { + OR: [ + { nodeOperationalStatus_isNull: true } + { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } + ] + } + } + ) { ...StorageBucketIds } } query getStorageBucketsByWorkerId($workerId: Int!) { storageBuckets( - where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } } + where: { + operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } + operatorMetadata: { + OR: [ + { nodeOperationalStatus_isNull: true } + { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } + ] + } + } ) { ...StorageBucketIds } } +query getStorageBucketsOperationalStatus($ids: [String!]) { + storageBuckets(where: { id_in: $ids }) { + id + operatorMetadata { + nodeOperationalStatus { + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + } + ... on NodeOperationalStatusNoServiceDuring { + __typename + forced + } + } + } + } +} + fragment StorageBucketDetails on StorageBucket { id operatorMetadata { diff --git a/storage-node/src/services/webApi/controllers/stateApi.ts b/storage-node/src/services/webApi/controllers/stateApi.ts index 75f85aa958..7d4932a404 100644 --- a/storage-node/src/services/webApi/controllers/stateApi.ts +++ b/storage-node/src/services/webApi/controllers/stateApi.ts @@ -147,6 +147,18 @@ export async function getStatus(req: express.Request, res: express.Response ({ + bucketId: id, + status: operatorMetadata?.nodeOperationalStatus?.__typename || 'NodeOperationalStatusNormal', + isForced: + operatorMetadata?.nodeOperationalStatus?.__typename === 'NodeOperationalStatusNormal' + ? false + : operatorMetadata?.nodeOperationalStatus?.forced || false, + }) + ) + res.status(200).json({ version: proc.version, uploadBuckets, @@ -154,6 +166,7 @@ export async function getStatus(req: express.Request, res: express.Response Date: Thu, 29 Feb 2024 11:28:04 +0500 Subject: [PATCH 13/18] update colossus docs --- storage-node/README.md | 186 ++++++++++++++++++++++++++--------------- 1 file changed, 117 insertions(+), 69 deletions(-) diff --git a/storage-node/README.md b/storage-node/README.md index f86147dbed..4b85d95866 100644 --- a/storage-node/README.md +++ b/storage-node/README.md @@ -6,17 +6,18 @@ Joystream storage node. ![License](https://img.shields.io/github/license/Joystream/joystream) -* [Colossus](#colossus) -* [Description](#description) -* [Installation](#installation) -* [Ubuntu Linux](#ubuntu-linux) -* [Install packages required for installation](#install-packages-required-for-installation) -* [Clone the code repository](#clone-the-code-repository) -* [Install volta](#install-volta) -* [Install project dependencies and build it](#install-project-dependencies-and-build-it) -* [Verify installation](#verify-installation) -* [Usage](#usage) -* [CLI Commands](#cli-commands) + +- [Colossus](#colossus) +- [Description](#description) +- [Installation](#installation) +- [Ubuntu Linux](#ubuntu-linux) +- [Install packages required for installation](#install-packages-required-for-installation) +- [Clone the code repository](#clone-the-code-repository) +- [Install volta](#install-volta) +- [Install project dependencies and build it](#install-project-dependencies-and-build-it) +- [Verify installation](#verify-installation) +- [Usage](#usage) +- [CLI Commands](#cli-commands) # Description @@ -147,29 +148,31 @@ There is also an option to run Colossus as [Docker container](../colossus.Docker # CLI Commands -* [`storage-node help [COMMAND]`](#storage-node-help-command) -* [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) -* [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) -* [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) -* [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) -* [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) -* [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) -* [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) -* [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) -* [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) -* [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) -* [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) -* [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) -* [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) -* [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) -* [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) -* [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) -* [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) -* [`storage-node server`](#storage-node-server) -* [`storage-node util:cleanup`](#storage-node-utilcleanup) -* [`storage-node util:fetch-bucket`](#storage-node-utilfetch-bucket) -* [`storage-node util:multihash`](#storage-node-utilmultihash) -* [`storage-node util:verify-bag-id`](#storage-node-utilverify-bag-id) + +- [`storage-node help [COMMAND]`](#storage-node-help-command) +- [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) +- [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) +- [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) +- [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) +- [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) +- [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) +- [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) +- [`storage-node leader:set-node-operational-status`](#storage-node-leaderset-node-operational-status) +- [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) +- [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) +- [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) +- [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) +- [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) +- [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) +- [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) +- [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) +- [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) +- [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) +- [`storage-node server`](#storage-node-server) +- [`storage-node util:cleanup`](#storage-node-utilcleanup) +- [`storage-node util:fetch-bucket`](#storage-node-utilfetch-bucket) +- [`storage-node util:multihash`](#storage-node-utilmultihash) +- [`storage-node util:verify-bag-id`](#storage-node-utilverify-bag-id) ## `storage-node help [COMMAND]` @@ -213,7 +216,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/cancel-invite.ts)_ +_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/cancel-invite.ts)_ ## `storage-node leader:create-bucket` @@ -244,7 +247,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/create-bucket.ts)_ +_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/create-bucket.ts)_ ## `storage-node leader:delete-bucket` @@ -271,7 +274,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/delete-bucket.ts)_ +_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/delete-bucket.ts)_ ## `storage-node leader:invite-operator` @@ -300,7 +303,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/invite-operator.ts)_ +_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/invite-operator.ts)_ ## `storage-node leader:remove-operator` @@ -327,7 +330,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/remove-operator.ts)_ +_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/remove-operator.ts)_ ## `storage-node leader:set-bucket-limits` @@ -357,7 +360,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/set-bucket-limits.ts)_ +_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-bucket-limits.ts)_ ## `storage-node leader:set-global-uploading-status` @@ -385,7 +388,44 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/set-global-uploading-status.ts)_ +_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-global-uploading-status.ts)_ + +## `storage-node leader:set-node-operational-status` + +Set/update storage node operational status. Requires storage working group leader permissions. + +``` +USAGE + $ storage-node leader:set-node-operational-status + +OPTIONS + -h, --help show CLI help + -i, --bucketId=bucketId (required) Storage bucket ID + -k, --keyFile=keyFile Path to key file to add to the keyring. + -m, --dev Use development mode + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator + + -p, --password=password Password to unlock keyfiles. Multiple + passwords can be passed, to try against all + files. If not specified a single password + can be set in ACCOUNT_PWD environment + variable. + + -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API + URL. Mandatory in non-dev environment. + + -w, --workerId=workerId (required) ID of the operator (storage group + worker) + + -y, --accountUri=accountUri Account URI (optional). If not specified a + single key can be set in ACCOUNT_URI + environment variable. + + --keyStore=keyStore Path to a folder with multiple key files to + load into keystore. +``` + +_See code: [src/commands/leader/set-node-operational-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/set-node-operational-status.ts)_ ## `storage-node leader:update-bag-limit` @@ -412,7 +452,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bag-limit.ts)_ +_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bag-limit.ts)_ ## `storage-node leader:update-bags` @@ -468,7 +508,7 @@ OPTIONS Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bags.ts)_ +_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bags.ts)_ ## `storage-node leader:update-blacklist` @@ -497,7 +537,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-blacklist.ts)_ +_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-blacklist.ts)_ ## `storage-node leader:update-bucket-status` @@ -526,7 +566,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-bucket-status.ts)_ +_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-bucket-status.ts)_ ## `storage-node leader:update-data-fee` @@ -553,7 +593,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-data-fee.ts)_ +_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-data-fee.ts)_ ## `storage-node leader:update-data-object-bloat-bond` @@ -581,7 +621,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-data-object-bloat-bond.ts)_ +_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-data-object-bloat-bond.ts)_ ## `storage-node leader:update-dynamic-bag-policy` @@ -611,7 +651,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-dynamic-bag-policy.ts)_ +_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-dynamic-bag-policy.ts)_ ## `storage-node leader:update-voucher-limits` @@ -640,7 +680,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/leader/update-voucher-limits.ts)_ +_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/leader/update-voucher-limits.ts)_ ## `storage-node operator:accept-invitation` @@ -673,7 +713,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/operator/accept-invitation.ts)_ +_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/operator/accept-invitation.ts)_ ## `storage-node operator:set-metadata` @@ -684,27 +724,34 @@ USAGE $ storage-node operator:set-metadata OPTIONS - -e, --endpoint=endpoint Root distribution node endpoint - -h, --help show CLI help - -i, --bucketId=bucketId (required) Storage bucket ID - -j, --jsonFile=jsonFile Path to JSON metadata file - -k, --keyFile=keyFile Path to key file to add to the keyring. - -m, --dev Use development mode + -e, --endpoint=endpoint Root distribution node endpoint + -h, --help show CLI help + -i, --bucketId=bucketId (required) Storage bucket ID + -j, --jsonFile=jsonFile Path to JSON metadata file + -k, --keyFile=keyFile Path to key file to add to the keyring. + -m, --dev Use development mode + -o, --operationalStatus=(Normal|NoService|NoServiceFrom|NoServiceDuring) Operational status of the operator - -p, --password=password Password to unlock keyfiles. Multiple passwords can be passed, to try against all files. - If not specified a single password can be set in ACCOUNT_PWD environment variable. + -p, --password=password Password to unlock keyfiles. Multiple + passwords can be passed, to try against all + files. If not specified a single password + can be set in ACCOUNT_PWD environment + variable. - -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment. + -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API + URL. Mandatory in non-dev environment. - -w, --workerId=workerId (required) Storage operator worker ID + -w, --workerId=workerId (required) Storage operator worker ID - -y, --accountUri=accountUri Account URI (optional). If not specified a single key can be set in ACCOUNT_URI - environment variable. + -y, --accountUri=accountUri Account URI (optional). If not specified a + single key can be set in ACCOUNT_URI + environment variable. - --keyStore=keyStore Path to a folder with multiple key files to load into keystore. + --keyStore=keyStore Path to a folder with multiple key files to + load into keystore. ``` -_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/operator/set-metadata.ts)_ +_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/operator/set-metadata.ts)_ ## `storage-node server` @@ -806,7 +853,7 @@ OPTIONS directory will be used. ``` -_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/server.ts)_ +_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/server.ts)_ ## `storage-node util:cleanup` @@ -844,7 +891,7 @@ OPTIONS --keyStore=keyStore Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/cleanup.ts)_ +_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/cleanup.ts)_ ## `storage-node util:fetch-bucket` @@ -877,7 +924,7 @@ OPTIONS under the uploads directory will be used. ``` -_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/fetch-bucket.ts)_ +_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/fetch-bucket.ts)_ ## `storage-node util:multihash` @@ -892,7 +939,7 @@ OPTIONS -h, --help show CLI help ``` -_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/multihash.ts)_ +_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/multihash.ts)_ ## `storage-node util:verify-bag-id` @@ -920,5 +967,6 @@ OPTIONS - dynamic:member:4 ``` -_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v3.10.2/src/commands/util/verify-bag-id.ts)_ +_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v4.1.0/src/commands/util/verify-bag-id.ts)_ + From b6329f7f630a46b6d866538c7d13c21b456e70bd Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 11:28:57 +0500 Subject: [PATCH 14/18] [Argus] ensure distributor node only connects with operationally active storage nodes --- .../networking/query-node/queries/queries.graphql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/distributor-node/src/services/networking/query-node/queries/queries.graphql b/distributor-node/src/services/networking/query-node/queries/queries.graphql index e9c0254852..e7d1259b88 100644 --- a/distributor-node/src/services/networking/query-node/queries/queries.graphql +++ b/distributor-node/src/services/networking/query-node/queries/queries.graphql @@ -107,7 +107,17 @@ fragment StorageBucketOperatorFields on StorageBucket { } query getActiveStorageBucketOperatorsData { - storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) { + storageBuckets( + where: { + operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } + operatorMetadata: { + OR: [ + { nodeOperationalStatus_isNull: true } + { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } + ] + } + } + ) { ...StorageBucketOperatorFields } } From 172bcb749260d17fa0b868e991b94582474d76bb Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 11:32:51 +0500 Subject: [PATCH 15/18] revert QN changes (i.e. remove mappings and schema changes) --- query-node/CHANGELOG.md | 4 - query-node/mappings/package.json | 2 +- query-node/mappings/src/storage/index.ts | 105 +------------------- query-node/mappings/src/storage/metadata.ts | 95 +++--------------- query-node/mappings/src/workingGroups.ts | 3 - query-node/package.json | 2 +- query-node/schemas/storage.graphql | 50 ---------- query-node/schemas/storageEvents.graphql | 53 ---------- 8 files changed, 18 insertions(+), 296 deletions(-) delete mode 100644 query-node/schemas/storageEvents.graphql diff --git a/query-node/CHANGELOG.md b/query-node/CHANGELOG.md index 243898dd85..710056e7e7 100644 --- a/query-node/CHANGELOG.md +++ b/query-node/CHANGELOG.md @@ -1,7 +1,3 @@ -### 1.10.0 - -- Updates mappings to process `NodeOperationalStatusMetadata` protobuf message. This metaprotocol message enables both storage/distribution workers & leads to set the operational status of the nodes. - ### 1.9.0 - Add `isShort` field to `Video` entity. and updated mappings to process/set this filed diff --git a/query-node/mappings/package.json b/query-node/mappings/package.json index 8dc5ea5fd6..2449553194 100644 --- a/query-node/mappings/package.json +++ b/query-node/mappings/package.json @@ -1,6 +1,6 @@ { "name": "query-node-mappings", - "version": "1.10.0", + "version": "1.9.0", "description": "Mappings for hydra-processor", "main": "lib/src/index.js", "license": "MIT", diff --git a/query-node/mappings/src/storage/index.ts b/query-node/mappings/src/storage/index.ts index d4f1fdfda6..9f5224b58e 100644 --- a/query-node/mappings/src/storage/index.ts +++ b/query-node/mappings/src/storage/index.ts @@ -1,9 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { DatabaseManager, EventContext, StoreContext, SubstrateEvent } from '@joystream/hydra-common' -import { ISetNodeOperationalStatus } from '@joystream/metadata-protobuf' -import { isSet } from '@joystream/metadata-protobuf/utils' +import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common' import BN from 'bn.js' import { DistributionBucket, @@ -11,19 +9,14 @@ import { DistributionBucketOperator, DistributionBucketOperatorMetadata, DistributionBucketOperatorStatus, - DistributionNodeOperationalStatusSetEvent, GeoCoordinates, NodeLocationMetadata, StorageBag, StorageBucket, - StorageBucketOperatorMetadata, StorageBucketOperatorStatusActive, StorageBucketOperatorStatusInvited, StorageBucketOperatorStatusMissing, StorageDataObject, - StorageNodeOperationalStatusSetEvent, - Worker, - WorkingGroup, } from 'query-node/dist/model' import { Storage_DataObjectsDeletedEvent_V1001 as DataObjectsDeletedEvent_V1001, @@ -57,20 +50,12 @@ import { Storage_StorageOperatorMetadataSetEvent_V1001 as StorageOperatorMetadataSetEvent_V1001, Storage_VoucherChangedEvent_V1001 as VoucherChangedEvent_V1001, } from '../../generated/types' -import { - RelationsArr, - deterministicEntityId, - genericEventFields, - getById, - getByIdOrFail, - invalidMetadata, -} from '../common' +import { RelationsArr, getById, getByIdOrFail } from '../common' import { unsetAssetRelations, videoRelationsForCounters } from '../content/utils' import { getAllManagers } from '../derivedPropertiesManager/applications' import { processDistributionBucketFamilyMetadata, processDistributionOperatorMetadata, - processNodeOperationalStatusMetadata, processStorageOperatorMetadata, } from './metadata' import { @@ -617,89 +602,3 @@ async function removeDistributionBucketOperator(store: DatabaseManager, operator } } } - -export async function processSetNodeOperationalStatusMessage( - store: DatabaseManager, - event: SubstrateEvent, - workingGroup: WorkingGroup, - meta: ISetNodeOperationalStatus -): Promise { - if (workingGroup.name !== 'distributionWorkingGroup' && workingGroup.name !== 'storageWorkingGroup') { - return invalidMetadata(`The ${workingGroup.name} is incompatible with the remarked setNodeOperationalStatus`) - } - - const workerId = Number(meta.workerId) - const bucketId = String(meta.bucketId) - - if ((await getById(store, Worker, `${workingGroup.name}-${workerId}`)) === undefined) { - return invalidMetadata(`The worker ${workerId} does not exist in the ${workingGroup.name} working group`) - } - - // Update the operational status of Storage node - if (workingGroup.name === 'storageWorkingGroup') { - const storageBucket = await store.get(StorageBucket, { where: { id: bucketId }, relations: ['operatorMetadata'] }) - if (!storageBucket) { - return invalidMetadata(`The storage bucket ${bucketId} does not exist`) - } else if (storageBucket.operatorStatus.isTypeOf !== 'StorageBucketOperatorStatusActive') { - return invalidMetadata(`The storage bucket ${bucketId} is not active`) - } - - // create metadata entity if it does not exist already - const metadataEntity = - storageBucket.operatorMetadata || new StorageBucketOperatorMetadata({ id: deterministicEntityId(event) }) - - if (isSet(meta.operationalStatus)) { - metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( - 'lead', - metadataEntity.nodeOperationalStatus, - meta.operationalStatus - ) - } - await store.save(metadataEntity) - - // event processing - - const operationalStatusSetEvent = new StorageNodeOperationalStatusSetEvent({ - ...genericEventFields(event), - storageBucket, - operationalStatus: metadataEntity.nodeOperationalStatus, - }) - - await store.save(operationalStatusSetEvent) - } - - // Update the operational status of Distribution node - if (workingGroup.name === 'distributionWorkingGroup') { - const distributionOperatorId = `${bucketId}-${workerId}` - const operator = await store.get(DistributionBucketOperator, { - where: { id: distributionOperatorId }, - relations: ['metadata'], - }) - if (!operator) { - return invalidMetadata(`The distribution bucket operator ${distributionOperatorId} does not exist`) - } - - // create metadata entity if it does not exist already - const metadataEntity = - operator.metadata || new DistributionBucketOperatorMetadata({ id: deterministicEntityId(event) }) - - if (isSet(meta.operationalStatus)) { - metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( - 'lead', - metadataEntity.nodeOperationalStatus, - meta.operationalStatus - ) - } - await store.save(metadataEntity) - - // event processing - - const operationalStatusSetEvent = new DistributionNodeOperationalStatusSetEvent({ - ...genericEventFields(event), - bucketOperator: operator, - operationalStatus: metadataEntity.nodeOperationalStatus, - }) - - await store.save(operationalStatusSetEvent) - } -} diff --git a/query-node/mappings/src/storage/metadata.ts b/query-node/mappings/src/storage/metadata.ts index 6d7a8f8f5a..f18fe29724 100644 --- a/query-node/mappings/src/storage/metadata.ts +++ b/query-node/mappings/src/storage/metadata.ts @@ -1,34 +1,27 @@ import { DatabaseManager, SubstrateEvent } from '@joystream/hydra-common' import { - DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto, - DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto, - GeographicalArea as GeographicalAreaProto, - INodeLocationMetadata, - INodeOperationalStatusMetadata, - NodeOperationalStatusMetadata, - StorageBucketOperatorMetadata as StorageBucketOperatorMetadataProto, -} from '@joystream/metadata-protobuf' -import { isEmptyObject, isSet, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils' -import { Bytes } from '@polkadot/types' -import _ from 'lodash' -import { - Continent, - DistributionBucketFamilyGeographicArea, DistributionBucketFamilyMetadata, DistributionBucketOperatorMetadata, + StorageBucketOperatorMetadata, GeoCoordinates, + NodeLocationMetadata, + Continent, GeographicalAreaContinent, GeographicalAreaCountry, GeographicalAreaSubdivistion, - NodeLocationMetadata, - NodeOperationalStatus, - NodeOperationalStatusNoService, - NodeOperationalStatusNoServiceDuring, - NodeOperationalStatusNoServiceFrom, - NodeOperationalStatusNormal, - StorageBucketOperatorMetadata, + DistributionBucketFamilyGeographicArea, } from 'query-node/dist/model' import { deserializeMetadata, deterministicEntityId, invalidMetadata } from '../common' +import { Bytes } from '@polkadot/types' +import { + DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto, + StorageBucketOperatorMetadata as StorageBucketOperatorMetadataProto, + DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto, + INodeLocationMetadata, + GeographicalArea as GeographicalAreaProto, +} from '@joystream/metadata-protobuf' +import { isSet, isEmptyObject, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils' +import _ from 'lodash' const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Continent]: Continent } = { [GeographicalAreaProto.Continent.AF]: Continent.AF, @@ -73,52 +66,6 @@ async function processNodeLocationMetadata( return nodeLocation } -export function processNodeOperationalStatusMetadata( - actorContext: 'lead' | 'worker', - current: typeof NodeOperationalStatus | undefined, - meta: INodeOperationalStatusMetadata -): typeof NodeOperationalStatus { - const isCurrentForced = - current && - (current instanceof NodeOperationalStatusNoService || - current instanceof NodeOperationalStatusNoServiceFrom || - current instanceof NodeOperationalStatusNoServiceDuring) && - current.forced - - // if current state is forced by lead, then prevent worker from unilaterally reversing. - if (isCurrentForced && actorContext === 'worker') { - return current - } - - // set node state to NoService - if (meta.status === NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE) { - if (meta.noServiceFrom && meta.noServiceTo) { - const status = new NodeOperationalStatusNoServiceDuring() - status.from = new Date(meta.noServiceFrom) - status.to = new Date(meta.noServiceTo) - status.rationale = meta.rationale || (null as any) - status.forced = actorContext === 'lead' - return status - } else if (meta.noServiceFrom && !meta.noServiceTo) { - const status = new NodeOperationalStatusNoServiceFrom() - status.from = new Date(meta.noServiceFrom) - status.rationale = meta.rationale || (null as any) - status.forced = actorContext === 'lead' - return status - } else if (!meta.noServiceFrom && !meta.noServiceTo) { - const status = new NodeOperationalStatusNoService() - status.rationale = meta.rationale || (null as any) - status.forced = actorContext === 'lead' - return status - } - } - - // Default operational status of the node - const status = new NodeOperationalStatusNormal() - status.rationale = meta.rationale || (null as any) - return status -} - export async function processDistributionOperatorMetadata( event: SubstrateEvent, store: DatabaseManager, @@ -141,13 +88,6 @@ export async function processDistributionOperatorMetadata( if (isSet(meta.extra)) { metadataEntity.extra = meta.extra } - if (isSet(meta.operationalStatus)) { - metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( - 'worker', - metadataEntity.nodeOperationalStatus, - meta.operationalStatus - ) - } await store.save(metadataEntity) @@ -176,13 +116,6 @@ export async function processStorageOperatorMetadata( if (isSet(meta.extra)) { metadataEntity.extra = meta.extra || (null as any) } - if (isSet(meta.operationalStatus)) { - metadataEntity.nodeOperationalStatus = processNodeOperationalStatusMetadata( - 'worker', - metadataEntity.nodeOperationalStatus, - meta.operationalStatus - ) - } await store.save(metadataEntity) diff --git a/query-node/mappings/src/workingGroups.ts b/query-node/mappings/src/workingGroups.ts index 1d85dfbdfe..1c87261894 100644 --- a/query-node/mappings/src/workingGroups.ts +++ b/query-node/mappings/src/workingGroups.ts @@ -118,7 +118,6 @@ import { toNumber, } from './common' import { moderatePost } from './forum' -import { processSetNodeOperationalStatusMessage } from './storage' // Reusable functions async function getWorkingGroupLeadOrFail(store: DatabaseManager, groupName: WorkingGroupModuleName): Promise { @@ -450,8 +449,6 @@ async function applyWorkingGroupsRemark( member.metadata.isVerifiedValidator = isVerified await store.save(member.metadata) await store.save(member) - } else if (metadata?.setNodeOperationalStatus) { - await processSetNodeOperationalStatusMessage(store, event, group, metadata.setNodeOperationalStatus) } else { return invalidMetadata('Unrecognized remarked action') } diff --git a/query-node/package.json b/query-node/package.json index 1b8bdbce14..22a2aadbb6 100644 --- a/query-node/package.json +++ b/query-node/package.json @@ -1,6 +1,6 @@ { "name": "query-node-root", - "version": "1.10.0", + "version": "1.9.0", "description": "GraphQL server and mappings. Generated with ♥ by Hydra-CLI", "scripts": { "build": "./build.sh", diff --git a/query-node/schemas/storage.graphql b/query-node/schemas/storage.graphql index 71afee25b5..c38028fcdb 100644 --- a/query-node/schemas/storage.graphql +++ b/query-node/schemas/storage.graphql @@ -58,50 +58,6 @@ type NodeLocationMetadata @entity { coordinates: GeoCoordinates } -type NodeOperationalStatusNormal @variant { - "Reason why node was set to this state" - rationale: String -} - -type NodeOperationalStatusNoService @variant { - "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." - forced: Boolean! - - "Reason why node was set to this state" - rationale: String -} - -type NodeOperationalStatusNoServiceFrom @variant { - "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." - forced: Boolean! - - "The time from which the bucket would have to no service" - from: DateTime! - - "Reason why node was set to this state" - rationale: String -} - -type NodeOperationalStatusNoServiceDuring @variant { - "Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing." - forced: Boolean! - - "The time from which the bucket would have to no service" - from: DateTime! - - "The time until which the bucket would have to no service" - to: DateTime! - - "Reason why node was set to this state" - rationale: String -} - -union NodeOperationalStatus = - NodeOperationalStatusNormal - | NodeOperationalStatusNoService - | NodeOperationalStatusNoServiceFrom - | NodeOperationalStatusNoServiceDuring - type StorageBucketOperatorMetadata @entity { "Root node endpoint" nodeEndpoint: String @@ -109,9 +65,6 @@ type StorageBucketOperatorMetadata @entity { "Optional node location metadata" nodeLocation: NodeLocationMetadata - "Optional node operational status" - nodeOperationalStatus: NodeOperationalStatus - "Additional information about the node/operator" extra: String } @@ -308,9 +261,6 @@ type DistributionBucketOperatorMetadata @entity { "Optional node location metadata" nodeLocation: NodeLocationMetadata - "Optional node operational status" - nodeOperationalStatus: NodeOperationalStatus - "Additional information about the node/operator" extra: String } diff --git a/query-node/schemas/storageEvents.graphql b/query-node/schemas/storageEvents.graphql deleted file mode 100644 index 724e28e08f..0000000000 --- a/query-node/schemas/storageEvents.graphql +++ /dev/null @@ -1,53 +0,0 @@ -type StorageNodeOperationalStatusSetEvent implements Event @entity { - ### GENERIC DATA ### - - "(network}-{blockNumber}-{indexInBlock}" - id: ID! - - "Hash of the extrinsic which caused the event to be emitted" - inExtrinsic: String - - "Blocknumber of the block in which the event was emitted." - inBlock: Int! - - "Network the block was produced in" - network: Network! - - "Index of event in block from which it was emitted." - indexInBlock: Int! - - ### SPECIFIC DATA ### - - "Storage Bucket" - storageBucket: StorageBucket - - "Operational status that was set" - operationalStatus: NodeOperationalStatus! -} - -type DistributionNodeOperationalStatusSetEvent implements Event @entity { - ### GENERIC DATA ### - - "(network}-{blockNumber}-{indexInBlock}" - id: ID! - - "Hash of the extrinsic which caused the event to be emitted" - inExtrinsic: String - - "Blocknumber of the block in which the event was emitted." - inBlock: Int! - - "Network the block was produced in" - network: Network! - - "Index of event in block from which it was emitted." - indexInBlock: Int! - - ### SPECIFIC DATA ### - - "Distribution bucket operator" - bucketOperator: DistributionBucketOperator - - "Operational status that was set" - operationalStatus: NodeOperationalStatus! -} From 16aaa48026a9cb555e6886502e4fdf2a9b8ea45a Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 29 Feb 2024 15:16:29 +0500 Subject: [PATCH 16/18] update graphql queries --- .../query-node/queries/queries.graphql | 2 +- .../networking/query-node/schema.graphql | 54 ++++++++++++------- storage-node/src/services/queryNode/api.ts | 2 +- .../queryNode/queries/queries.graphql | 2 +- .../src/services/queryNode/schema.graphql | 54 ++++++++++++------- .../src/services/sync/storageObligations.ts | 2 +- 6 files changed, 74 insertions(+), 42 deletions(-) diff --git a/distributor-node/src/services/networking/query-node/queries/queries.graphql b/distributor-node/src/services/networking/query-node/queries/queries.graphql index e7d1259b88..11297acf42 100644 --- a/distributor-node/src/services/networking/query-node/queries/queries.graphql +++ b/distributor-node/src/services/networking/query-node/queries/queries.graphql @@ -93,7 +93,7 @@ query getDistributionBucketsWithBagsByIds($ids: [String!]) { } } -query getDistributionBucketsWithBagsByWorkerId($workerId: Int!) { +query getDistributionBucketsWithBagsByWorkerId($workerId: BigInt!) { distributionBuckets(where: { operators_some: { workerId_eq: $workerId, status_eq: ACTIVE } }) { ...DistributionBucketWithBags } diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql index c0ba380b15..9a5be3c8a3 100644 --- a/distributor-node/src/services/networking/query-node/schema.graphql +++ b/distributor-node/src/services/networking/query-node/schema.graphql @@ -347,7 +347,7 @@ type DistributionBucketOperator { distributionBucket: DistributionBucket! """ID of the distribution group worker""" - workerId: Int! + workerId: BigInt! """Current operator status""" status: DistributionBucketOperatorStatus! @@ -583,14 +583,14 @@ input DistributionBucketOperatorWhereInput { distributionBucket_isNull: Boolean distributionBucket: DistributionBucketWhereInput workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] status_isNull: Boolean status_eq: DistributionBucketOperatorStatus status_not_eq: DistributionBucketOperatorStatus @@ -1676,12 +1676,12 @@ input StorageBucketOperatorMetadataWhereInput { union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive type StorageBucketOperatorStatusActive { - workerId: Int! + workerId: BigInt! transactorAccountId: String! } type StorageBucketOperatorStatusInvited { - workerId: Int! + workerId: BigInt! } type StorageBucketOperatorStatusMissing { @@ -1699,14 +1699,14 @@ input StorageBucketOperatorStatusWhereInput { phantom_in: [Int!] phantom_not_in: [Int!] workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] transactorAccountId_isNull: Boolean transactorAccountId_eq: String transactorAccountId_not_eq: String @@ -2152,6 +2152,9 @@ input WhereIdInput { type Worker { """Worker id ({workingGroupName}-{workerId})""" id: String! + + """WorkerId in specific working group module""" + runtimeId: BigInt! } type WorkerEdge { @@ -2164,6 +2167,10 @@ enum WorkerOrderByInput { id_DESC id_ASC_NULLS_FIRST id_DESC_NULLS_LAST + runtimeId_ASC + runtimeId_DESC + runtimeId_ASC_NULLS_FIRST + runtimeId_DESC_NULLS_LAST } type WorkersConnection { @@ -2190,6 +2197,15 @@ input WorkerWhereInput { id_not_startsWith: String id_endsWith: String id_not_endsWith: String + runtimeId_isNull: Boolean + runtimeId_eq: BigInt + runtimeId_not_eq: BigInt + runtimeId_gt: BigInt + runtimeId_gte: BigInt + runtimeId_lt: BigInt + runtimeId_lte: BigInt + runtimeId_in: [BigInt!] + runtimeId_not_in: [BigInt!] AND: [WorkerWhereInput!] OR: [WorkerWhereInput!] } diff --git a/storage-node/src/services/queryNode/api.ts b/storage-node/src/services/queryNode/api.ts index 863474bcda..43fee13be9 100644 --- a/storage-node/src/services/queryNode/api.ts +++ b/storage-node/src/services/queryNode/api.ts @@ -187,7 +187,7 @@ export class QueryNodeApi { * * @param workerId - worker ID */ - public async getStorageBucketIdsByWorkerId(workerId: number): Promise> { + public async getStorageBucketIdsByWorkerId(workerId: bigint): Promise> { const result = await this.multipleEntitiesQuery< GetStorageBucketsByWorkerIdQuery, GetStorageBucketsByWorkerIdQueryVariables diff --git a/storage-node/src/services/queryNode/queries/queries.graphql b/storage-node/src/services/queryNode/queries/queries.graphql index d2274e2180..b7ceed17b7 100644 --- a/storage-node/src/services/queryNode/queries/queries.graphql +++ b/storage-node/src/services/queryNode/queries/queries.graphql @@ -18,7 +18,7 @@ query getStorageBuckets { } } -query getStorageBucketsByWorkerId($workerId: Int!) { +query getStorageBucketsByWorkerId($workerId: BigInt!) { storageBuckets( where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } diff --git a/storage-node/src/services/queryNode/schema.graphql b/storage-node/src/services/queryNode/schema.graphql index c0ba380b15..9a5be3c8a3 100644 --- a/storage-node/src/services/queryNode/schema.graphql +++ b/storage-node/src/services/queryNode/schema.graphql @@ -347,7 +347,7 @@ type DistributionBucketOperator { distributionBucket: DistributionBucket! """ID of the distribution group worker""" - workerId: Int! + workerId: BigInt! """Current operator status""" status: DistributionBucketOperatorStatus! @@ -583,14 +583,14 @@ input DistributionBucketOperatorWhereInput { distributionBucket_isNull: Boolean distributionBucket: DistributionBucketWhereInput workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] status_isNull: Boolean status_eq: DistributionBucketOperatorStatus status_not_eq: DistributionBucketOperatorStatus @@ -1676,12 +1676,12 @@ input StorageBucketOperatorMetadataWhereInput { union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | StorageBucketOperatorStatusInvited | StorageBucketOperatorStatusActive type StorageBucketOperatorStatusActive { - workerId: Int! + workerId: BigInt! transactorAccountId: String! } type StorageBucketOperatorStatusInvited { - workerId: Int! + workerId: BigInt! } type StorageBucketOperatorStatusMissing { @@ -1699,14 +1699,14 @@ input StorageBucketOperatorStatusWhereInput { phantom_in: [Int!] phantom_not_in: [Int!] workerId_isNull: Boolean - workerId_eq: Int - workerId_not_eq: Int - workerId_gt: Int - workerId_gte: Int - workerId_lt: Int - workerId_lte: Int - workerId_in: [Int!] - workerId_not_in: [Int!] + workerId_eq: BigInt + workerId_not_eq: BigInt + workerId_gt: BigInt + workerId_gte: BigInt + workerId_lt: BigInt + workerId_lte: BigInt + workerId_in: [BigInt!] + workerId_not_in: [BigInt!] transactorAccountId_isNull: Boolean transactorAccountId_eq: String transactorAccountId_not_eq: String @@ -2152,6 +2152,9 @@ input WhereIdInput { type Worker { """Worker id ({workingGroupName}-{workerId})""" id: String! + + """WorkerId in specific working group module""" + runtimeId: BigInt! } type WorkerEdge { @@ -2164,6 +2167,10 @@ enum WorkerOrderByInput { id_DESC id_ASC_NULLS_FIRST id_DESC_NULLS_LAST + runtimeId_ASC + runtimeId_DESC + runtimeId_ASC_NULLS_FIRST + runtimeId_DESC_NULLS_LAST } type WorkersConnection { @@ -2190,6 +2197,15 @@ input WorkerWhereInput { id_not_startsWith: String id_endsWith: String id_not_endsWith: String + runtimeId_isNull: Boolean + runtimeId_eq: BigInt + runtimeId_not_eq: BigInt + runtimeId_gt: BigInt + runtimeId_gte: BigInt + runtimeId_lt: BigInt + runtimeId_lte: BigInt + runtimeId_in: [BigInt!] + runtimeId_not_in: [BigInt!] AND: [WorkerWhereInput!] OR: [WorkerWhereInput!] } diff --git a/storage-node/src/services/sync/storageObligations.ts b/storage-node/src/services/sync/storageObligations.ts index 86fa98ec15..ff51a7691e 100644 --- a/storage-node/src/services/sync/storageObligations.ts +++ b/storage-node/src/services/sync/storageObligations.ts @@ -130,7 +130,7 @@ export async function getStorageObligationsFromRuntime( * @returns storage bucket IDs */ export async function getStorageBucketIdsByWorkerId(qnApi: QueryNodeApi, workerId: number): Promise { - const idFragments = await qnApi.getStorageBucketIdsByWorkerId(workerId) + const idFragments = await qnApi.getStorageBucketIdsByWorkerId(BigInt(workerId)) const ids = idFragments.map((frag) => frag.id) return ids From 971aaaeafd2438d33c1201c3aaafa60a48817f28 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 27 Mar 2024 06:28:03 +0500 Subject: [PATCH 17/18] address requested changes --- distributor-node/src/command-base/default.ts | 2 +- .../leader/set-node-operational-status.ts | 50 +++++++----- .../src/commands/operator/set-metadata.ts | 47 +++++------ .../src/schemas/operatorMetadataSchema.ts | 81 +++++++++++++++++-- .../src/services/networking/runtime/api.ts | 14 ++-- distributor-node/src/types/metadata.ts | 5 -- metadata-protobuf/proto/Storage.proto | 51 +++++++----- .../src/command-base/ApiCommandBase.ts | 2 +- .../leader/set-node-operational-status.ts | 64 ++++++++------- .../src/commands/operator/set-metadata.ts | 45 +++++------ .../schemas/operatorMetadataSchema.ts | 81 +++++++++++++++++-- .../src/services/metadata/schemas/types.ts | 6 -- .../src/services/runtime/extrinsics.ts | 4 +- 13 files changed, 291 insertions(+), 161 deletions(-) delete mode 100644 distributor-node/src/types/metadata.ts diff --git a/distributor-node/src/command-base/default.ts b/distributor-node/src/command-base/default.ts index 26d98ee885..b4f7247f78 100644 --- a/distributor-node/src/command-base/default.ts +++ b/distributor-node/src/command-base/default.ts @@ -107,7 +107,7 @@ export default abstract class DefaultCommandBase extends Command { type: 'datepicker', name: 'result', clearable: true, - default: new Date('2017-09-28 17:36:05').toISOString(), + default: new Date().toISOString(), }, ]) diff --git a/distributor-node/src/commands/leader/set-node-operational-status.ts b/distributor-node/src/commands/leader/set-node-operational-status.ts index 14cdad9913..b95bd9e319 100644 --- a/distributor-node/src/commands/leader/set-node-operational-status.ts +++ b/distributor-node/src/commands/leader/set-node-operational-status.ts @@ -1,12 +1,11 @@ import { - INodeOperationalStatusMetadata, + INodeOperationalStatus, ISetNodeOperationalStatus, - NodeOperationalStatusMetadata, + NodeOperationalStatus, SetNodeOperationalStatus, } from '@joystream/metadata-protobuf' import AccountsCommandBase from '../../command-base/accounts' import DefaultCommandBase, { flags } from '../../command-base/default' -import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../types/metadata' export default class LeadSetNodeOperationalStatus extends AccountsCommandBase { static description = `Set/update distribution node operational status. Requires distribution working group leader permissions.` @@ -20,41 +19,54 @@ export default class LeadSetNodeOperationalStatus extends AccountsCommandBase { description: 'ID of the operator (distribution group worker)', required: true, }), - operationalStatus: flags.enum({ + operationalStatus: flags.enum>({ char: 'o', - options: [...NODE_OPERATIONAL_STATUS_OPTIONS], - required: false, + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: true, description: 'Operational status of the operator', }), + rationale: flags.string({ + char: 'r', + description: 'Rationale for setting the operational status', + }), ...DefaultCommandBase.flags, } async run(): Promise { - const { bucketId, workerId, operationalStatus: statusType } = this.parse(LeadSetNodeOperationalStatus).flags + const { + bucketId, + workerId, + rationale, + operationalStatus: statusType, + } = this.parse(LeadSetNodeOperationalStatus).flags const leadKey = await this.getDistributorLeadKey() - let operationalStatus: INodeOperationalStatusMetadata = {} + let operationalStatus: INodeOperationalStatus switch (statusType) { - case 'Normal': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + case 'normal': { + operationalStatus = { normal: { rationale } } break } - case 'NoService': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + case 'noService': { + operationalStatus = { noService: { rationale } } break } - case 'NoServiceFrom': { + case 'noServiceFrom': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceFrom: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, } break } - case 'NoServiceDuring': { + case 'noServiceUntil': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), - noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + noServiceUntil: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, } } } diff --git a/distributor-node/src/commands/operator/set-metadata.ts b/distributor-node/src/commands/operator/set-metadata.ts index 62bbb383b4..f1c0967597 100644 --- a/distributor-node/src/commands/operator/set-metadata.ts +++ b/distributor-node/src/commands/operator/set-metadata.ts @@ -1,14 +1,13 @@ import { DistributionBucketOperatorMetadata, IDistributionBucketOperatorMetadata, - INodeOperationalStatusMetadata, - NodeOperationalStatusMetadata, + INodeOperationalStatus, + NodeOperationalStatus, } from '@joystream/metadata-protobuf' import fs from 'fs' import AccountsCommandBase from '../../command-base/accounts' import DefaultCommandBase, { flags } from '../../command-base/default' import { ValidationService } from '../../services/validation/ValidationService' -import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../types/metadata' export default class OperatorSetMetadata extends AccountsCommandBase { static description = `Set/update distribution bucket operator metadata. @@ -28,9 +27,9 @@ export default class OperatorSetMetadata extends AccountsCommandBase { description: 'Root distribution node endpoint', exclusive: ['input'], }), - operationalStatus: flags.enum({ + operationalStatus: flags.enum>({ char: 'o', - options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], required: false, description: 'Operational status of the operator', }), @@ -46,28 +45,30 @@ export default class OperatorSetMetadata extends AccountsCommandBase { const { bucketId, workerId, input, endpoint, operationalStatus: statusType } = this.parse(OperatorSetMetadata).flags const workerKey = await this.getDistributorWorkerRoleKey(workerId) - let operationalStatus: INodeOperationalStatusMetadata = {} + let operationalStatus: INodeOperationalStatus switch (statusType) { - case 'Normal': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + case 'normal': { + operationalStatus = { normal: {} } break } - case 'NoService': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + case 'noService': { + operationalStatus = { noService: {} } break } - case 'NoServiceFrom': { + case 'noServiceFrom': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceFrom: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, } break } - case 'NoServiceDuring': { + case 'noServiceUntil': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), - noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + noServiceUntil: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, } } } @@ -78,15 +79,7 @@ export default class OperatorSetMetadata extends AccountsCommandBase { const params = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(input).toString())) metadata = { ...params, - operationalStatus: params.operationalStatus - ? { - ...params.operationalStatus, - status: - params.operationalStatus?.status === 'Normal' - ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL - : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - } - : {}, + ...(params.operationalStatus && { operationalStatus: params.operationalStatus }), } } else { metadata = { endpoint, operationalStatus } @@ -96,8 +89,8 @@ export default class OperatorSetMetadata extends AccountsCommandBase { bucketId: bucketId.toHuman(), workerId, metadata, - operationalStatus, }) + await this.sendAndFollowTx( await this.getDecodedPair(workerKey), this.api.tx.storage.setDistributionOperatorMetadata( diff --git a/distributor-node/src/schemas/operatorMetadataSchema.ts b/distributor-node/src/schemas/operatorMetadataSchema.ts index d9c2d759da..35ea8344cd 100644 --- a/distributor-node/src/schemas/operatorMetadataSchema.ts +++ b/distributor-node/src/schemas/operatorMetadataSchema.ts @@ -1,5 +1,4 @@ import { JSONSchema4 } from 'json-schema' -import { NODE_OPERATIONAL_STATUS_OPTIONS } from '../types/metadata' export const operatorMetadataSchema: JSONSchema4 = { type: 'object', @@ -23,13 +22,79 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, operationalStatus: { - type: 'object', - properties: { - status: { type: 'string', enum: [...NODE_OPERATIONAL_STATUS_OPTIONS] }, - noServiceFrom: { type: 'string', format: 'date-time' }, - noServiceTo: { type: 'string', format: 'date-time' }, - rationale: { type: 'string' }, - }, + oneOf: [ + { + type: 'object', + properties: { + normal: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['normal'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noService: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['noService'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceFrom: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + }, + required: ['from'], + }, + }, + required: ['noServiceFrom'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceUntil: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + until: { + type: 'string', + }, + }, + required: ['until'], + }, + }, + required: ['noServiceUntil'], + additionalProperties: false, + }, + ], }, extra: { type: 'string' }, }, diff --git a/distributor-node/src/services/networking/runtime/api.ts b/distributor-node/src/services/networking/runtime/api.ts index 1398b148ef..59731ddecf 100644 --- a/distributor-node/src/services/networking/runtime/api.ts +++ b/distributor-node/src/services/networking/runtime/api.ts @@ -1,13 +1,13 @@ -import { ApiPromise, WsProvider, SubmittableResult } from '@polkadot/api' -import { SubmittableExtrinsic, AugmentedEvent } from '@polkadot/api/types' +import { ApiPromise, SubmittableResult, WsProvider } from '@polkadot/api' +import { AugmentedEvent, SubmittableExtrinsic } from '@polkadot/api/types' import { KeyringPair } from '@polkadot/keyring/types' import { Balance } from '@polkadot/types/interfaces' -import { formatBalance } from '@polkadot/util' -import { IEvent } from '@polkadot/types/types' import { DispatchError } from '@polkadot/types/interfaces/system' -import { LoggingService } from '../../logging' -import { Logger } from 'winston' import { SpRuntimeDispatchError } from '@polkadot/types/lookup' +import { IEvent } from '@polkadot/types/types' +import { formatBalance } from '@polkadot/util' +import { Logger } from 'winston' +import { LoggingService } from '../../logging' export class ExtrinsicFailedError extends Error {} @@ -30,7 +30,7 @@ export class RuntimeApi { private static async initApi(apiUri: string) { const wsProvider: WsProvider = new WsProvider(apiUri) - const api = await ApiPromise.create({ provider: wsProvider }) + const api = await ApiPromise.create({ provider: wsProvider, throwOnConnect: true }) const [properties, chainType] = await Promise.all([api.rpc.system.properties(), api.rpc.system.chainType()]) diff --git a/distributor-node/src/types/metadata.ts b/distributor-node/src/types/metadata.ts deleted file mode 100644 index ad57d1c299..0000000000 --- a/distributor-node/src/types/metadata.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Distributor Node's operational states (are part of Operator metadata) -export const NODE_OPERATIONAL_STATUS_OPTIONS = ['Normal', 'NoService', 'NoServiceFrom', 'NoServiceDuring'] as const - -// convert NODE_OPERATIONAL_STATUS_OPTIONS into string literal union type -export type NodeOperationalStatus = typeof NODE_OPERATIONAL_STATUS_OPTIONS[number] diff --git a/metadata-protobuf/proto/Storage.proto b/metadata-protobuf/proto/Storage.proto index 8313dfa25a..d8407ad356 100644 --- a/metadata-protobuf/proto/Storage.proto +++ b/metadata-protobuf/proto/Storage.proto @@ -15,14 +15,14 @@ message StorageBucketOperatorMetadata { optional string endpoint = 1; // Root storage node endpoint (ie. https://example.com/storage) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator - optional NodeOperationalStatusMetadata operational_status = 4; // Node's operational status to set + optional NodeOperationalStatus operational_status = 4; // Node's operational status to set } message DistributionBucketOperatorMetadata { optional string endpoint = 1; // Root distribution node endpoint (ie. https://example.com/distribution) optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value) optional string extra = 3; // Additional information about the node / node operator - optional NodeOperationalStatusMetadata operational_status = 4; // Node's operational status to set + optional NodeOperationalStatus operational_status = 4; // Node's operational status to set } message GeographicalArea { @@ -49,30 +49,39 @@ message DistributionBucketFamilyMetadata { repeated string latency_test_targets = 4; // List of targets (hosts/ips) best suited latency measurements for this family } -message NodeOperationalStatusMetadata { - enum OperationalStatus { - // Node is operating normally - NORMAL = 0; +message NodeOperationalStatusNormal { + optional string rationale = 1; - // Node is not operational - NO_SERVICE = 1; - } +} + +message NodeOperationalStatusNoService { + optional string rationale = 1; + } - // Node's Operational status to set - optional OperationalStatus status = 1; +message NodeOperationalStatusNoServiceFrom { + optional string rationale = 1; + required string from = 2; // date +} - // Date (ISO format) from which the node won't be operational (Should be set if status is NoServiceFrom or NoServiceDuring) - optional string no_service_from = 2; +message NodeOperationalStatusNoServiceUntil { + optional string rationale = 1; + optional string from = 2; // date + required string until = 3; // date +} - // Date (ISO format) until which the node won't be operational (Should be set if status is NoServiceDuring) - optional string no_service_to = 3; - // Rationale for setting the current status - optional string rationale = 4; +message NodeOperationalStatus { + oneof node_operational_status { + NodeOperationalStatusNormal normal = 1; + NodeOperationalStatusNoService no_service = 2; + NodeOperationalStatusNoServiceFrom no_service_from = 3; + NodeOperationalStatusNoServiceUntil no_service_until = 4; + } } message SetNodeOperationalStatus { - optional NodeOperationalStatusMetadata operational_status = 1; // Node's operational status to set - optional string worker_id = 2; // Storage/Distribution Worker ID - optional string bucket_id = 3; // Storage/Distribution Bucket ID -} + required string worker_id = 1; // Storage/Distribution Worker ID + required string bucket_id = 2; // Storage/Distribution Bucket ID + required NodeOperationalStatus operational_status = 3; // Node's operational status to set + +} \ No newline at end of file diff --git a/storage-node/src/command-base/ApiCommandBase.ts b/storage-node/src/command-base/ApiCommandBase.ts index 3c1c2eb48e..c9ff367161 100644 --- a/storage-node/src/command-base/ApiCommandBase.ts +++ b/storage-node/src/command-base/ApiCommandBase.ts @@ -261,7 +261,7 @@ export default abstract class ApiCommandBase extends Command { type: 'datepicker', name: 'result', clearable: true, - default: new Date('2017-09-28 17:36:05').toISOString(), + default: new Date().toISOString(), }, ]) diff --git a/storage-node/src/commands/leader/set-node-operational-status.ts b/storage-node/src/commands/leader/set-node-operational-status.ts index b278860947..d978cf17e3 100644 --- a/storage-node/src/commands/leader/set-node-operational-status.ts +++ b/storage-node/src/commands/leader/set-node-operational-status.ts @@ -1,7 +1,6 @@ -import { INodeOperationalStatusMetadata, NodeOperationalStatusMetadata } from '@joystream/metadata-protobuf' +import { INodeOperationalStatus, NodeOperationalStatus } from '@joystream/metadata-protobuf' import { flags } from '@oclif/command' import LeaderCommandBase from '../../command-base/LeaderCommandBase' -import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../services/metadata/schemas' import { setStorageNodeOperationalStatus } from '../../services/runtime/extrinsics' /** @@ -26,63 +25,68 @@ export default class LeadSetNodeOperationalStatus extends LeaderCommandBase { description: 'ID of the operator (storage group worker)', required: true, }), - operationalStatus: flags.enum({ + operationalStatus: flags.enum>({ char: 'o', - options: [...NODE_OPERATIONAL_STATUS_OPTIONS], - required: false, + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], + required: true, description: 'Operational status of the operator', }), + rationale: flags.string({ + char: 'r', + description: 'Rationale for the operational status', + }), ...LeaderCommandBase.flags, } async run(): Promise { - const { flags } = this.parse(LeadSetNodeOperationalStatus) + const { + bucketId, + workerId, + rationale, + operationalStatus: statusType, + } = this.parse(LeadSetNodeOperationalStatus).flags const account = this.getAccount() const api = await this.getApi() - let operationalStatus: INodeOperationalStatusMetadata = {} - switch (flags.operationalStatus) { - case 'Normal': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + let operationalStatus: INodeOperationalStatus + switch (statusType) { + case 'normal': { + operationalStatus = { normal: { rationale } } break } - case 'NoService': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + case 'noService': { + operationalStatus = { noService: { rationale } } break } - case 'NoServiceFrom': { + case 'noServiceFrom': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceFrom: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, } break } - case 'NoServiceDuring': { + case 'noServiceUntil': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), - noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + noServiceUntil: { + rationale, + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, } } } this.log(`Setting node operational status...`, { - bucketId: flags.bucketId, - workerId: flags.workerId, + bucketId, + workerId, operationalStatus, }) - const success = await setStorageNodeOperationalStatus( - api, - account, - flags.workerId, - flags.bucketId, - operationalStatus - ) + const success = await setStorageNodeOperationalStatus(api, account, workerId, bucketId, operationalStatus) this.exitAfterRuntimeCall(success) - - this.log('Successfully set storage node operational status!') } } diff --git a/storage-node/src/commands/operator/set-metadata.ts b/storage-node/src/commands/operator/set-metadata.ts index ee38dd48b1..69371371d3 100644 --- a/storage-node/src/commands/operator/set-metadata.ts +++ b/storage-node/src/commands/operator/set-metadata.ts @@ -1,13 +1,12 @@ import { - INodeOperationalStatusMetadata, + INodeOperationalStatus, IStorageBucketOperatorMetadata, - NodeOperationalStatusMetadata, + NodeOperationalStatus, StorageBucketOperatorMetadata, } from '@joystream/metadata-protobuf' import { flags } from '@oclif/command' import fs from 'fs' import ApiCommandBase from '../../command-base/ApiCommandBase' -import { NODE_OPERATIONAL_STATUS_OPTIONS, NodeOperationalStatus } from '../../services/metadata/schemas' import { ValidationService } from '../../services/metadata/validationService' import { setStorageOperatorMetadata } from '../../services/runtime/extrinsics' import { getWorkerRoleAccount } from '../../services/runtime/queries' @@ -40,9 +39,9 @@ export default class OperatorSetMetadata extends ApiCommandBase { description: 'Root distribution node endpoint', exclusive: ['jsonFile'], }), - operationalStatus: flags.enum({ + operationalStatus: flags.enum>({ char: 'o', - options: [...NODE_OPERATIONAL_STATUS_OPTIONS], + options: ['normal', 'noService', 'noServiceFrom', 'noServiceUntil'], required: false, description: 'Operational status of the operator', }), @@ -58,28 +57,30 @@ export default class OperatorSetMetadata extends ApiCommandBase { const { flags } = this.parse(OperatorSetMetadata) const { workerId, bucketId, jsonFile, endpoint, operationalStatus: statusType } = flags - let operationalStatus: INodeOperationalStatusMetadata = {} + let operationalStatus: INodeOperationalStatus switch (statusType) { - case 'Normal': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NORMAL } + case 'normal': { + operationalStatus = { normal: {} } break } - case 'NoService': { - operationalStatus = { status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE } + case 'noService': { + operationalStatus = { noService: {} } break } - case 'NoServiceFrom': { + case 'noServiceFrom': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + noServiceFrom: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + }, } break } - case 'NoServiceDuring': { + case 'noServiceUntil': { operationalStatus = { - status: NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - noServiceFrom: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), - noServiceTo: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + noServiceUntil: { + from: (await this.datePrompt({ message: 'Enter No Service period start date' })).toISOString(), + until: (await this.datePrompt({ message: 'Enter No Service period end date' })).toISOString(), + }, } } } @@ -90,15 +91,7 @@ export default class OperatorSetMetadata extends ApiCommandBase { const input = validation.validate('OperatorMetadata', JSON.parse(fs.readFileSync(jsonFile).toString())) metadata = { ...input, - operationalStatus: input.operationalStatus - ? { - ...input.operationalStatus, - status: - input.operationalStatus?.status === 'Normal' - ? NodeOperationalStatusMetadata.OperationalStatus.NORMAL - : NodeOperationalStatusMetadata.OperationalStatus.NO_SERVICE, - } - : {}, + ...(input.operationalStatus && { operationalStatus: input.operationalStatus }), } } else { metadata = { endpoint, operationalStatus } diff --git a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts index fba41c78d8..5f11551d87 100644 --- a/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts +++ b/storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts @@ -1,5 +1,4 @@ import { JSONSchema4 } from 'json-schema' -import { NODE_OPERATIONAL_STATUS_OPTIONS } from './types' // Storage node operator metadata JSON schema. export const operatorMetadataSchema: JSONSchema4 = { @@ -24,13 +23,79 @@ export const operatorMetadataSchema: JSONSchema4 = { }, }, operationalStatus: { - type: 'object', - properties: { - status: { type: 'string', enum: [...NODE_OPERATIONAL_STATUS_OPTIONS] }, - noServiceFrom: { type: 'string', format: 'date-time' }, - noServiceTo: { type: 'string', format: 'date-time' }, - rationale: { type: 'string' }, - }, + oneOf: [ + { + type: 'object', + properties: { + normal: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['normal'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noService: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + }, + }, + }, + required: ['noService'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceFrom: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + }, + required: ['from'], + }, + }, + required: ['noServiceFrom'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + noServiceUntil: { + type: 'object', + properties: { + rationale: { + type: 'string', + }, + from: { + type: 'string', + }, + until: { + type: 'string', + }, + }, + required: ['until'], + }, + }, + required: ['noServiceUntil'], + additionalProperties: false, + }, + ], }, extra: { type: 'string' }, }, diff --git a/storage-node/src/services/metadata/schemas/types.ts b/storage-node/src/services/metadata/schemas/types.ts index ad49629ef4..bd02837513 100644 --- a/storage-node/src/services/metadata/schemas/types.ts +++ b/storage-node/src/services/metadata/schemas/types.ts @@ -4,9 +4,3 @@ import { schemas } from './schemas' export type SchemaKey = keyof typeof schemas & string export type TypeBySchemaKey = T extends 'OperatorMetadata' ? OperatorMetadataJson : never - -// Distributor Node's operational states (are part of Operator metadata) -export const NODE_OPERATIONAL_STATUS_OPTIONS = ['Normal', 'NoService', 'NoServiceFrom', 'NoServiceDuring'] as const - -// convert NODE_OPERATIONAL_STATUS_OPTIONS into string literal union type -export type NodeOperationalStatus = typeof NODE_OPERATIONAL_STATUS_OPTIONS[number] diff --git a/storage-node/src/services/runtime/extrinsics.ts b/storage-node/src/services/runtime/extrinsics.ts index 4f9d7c94a0..4aef7d2a03 100644 --- a/storage-node/src/services/runtime/extrinsics.ts +++ b/storage-node/src/services/runtime/extrinsics.ts @@ -1,5 +1,5 @@ import { - INodeOperationalStatusMetadata, + INodeOperationalStatus, ISetNodeOperationalStatus, SetNodeOperationalStatus, } from '@joystream/metadata-protobuf' @@ -600,7 +600,7 @@ export async function setStorageNodeOperationalStatus( account: KeyringPair, workerId: number, bucketId: number, - operationalStatus: INodeOperationalStatusMetadata + operationalStatus: INodeOperationalStatus ): Promise { return await extrinsicWrapper(() => { const metadata: ISetNodeOperationalStatus = { From 0537ee74c8f5caa8a14217668fc7cab59a5ca68c Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 27 Mar 2024 12:10:50 +0500 Subject: [PATCH 18/18] fix: only filter currently under maintenance nodes --- .../services/networking/NetworkingService.ts | 2 +- .../src/services/networking/query-node/api.ts | 23 +++- .../query-node/queries/queries.graphql | 32 +++-- .../networking/query-node/schema.graphql | 112 ++++-------------- storage-node/src/services/queryNode/api.ts | 27 ++++- .../queryNode/queries/queries.graphql | 54 +++++---- .../src/services/queryNode/schema.graphql | 112 ++++-------------- 7 files changed, 149 insertions(+), 213 deletions(-) diff --git a/distributor-node/src/services/networking/NetworkingService.ts b/distributor-node/src/services/networking/NetworkingService.ts index d2d5aba5bf..66c2e3a69f 100644 --- a/distributor-node/src/services/networking/NetworkingService.ts +++ b/distributor-node/src/services/networking/NetworkingService.ts @@ -402,7 +402,7 @@ export class NetworkingService { async checkActiveStorageNodeEndpoints(): Promise { try { - const activeStorageOperators = await this.queryNodeApi.getActiveStorageBucketOperatorsData() + const activeStorageOperators = await this.queryNodeApi.getOperationallyActiveStorageBucketOperatorsData() const endpoints = this.filterStorageNodeEndpoints( activeStorageOperators.map(({ id, operatorMetadata }) => ({ bucketId: id, diff --git a/distributor-node/src/services/networking/query-node/api.ts b/distributor-node/src/services/networking/query-node/api.ts index 6e769ed1f4..3609139fde 100644 --- a/distributor-node/src/services/networking/query-node/api.ts +++ b/distributor-node/src/services/networking/query-node/api.ts @@ -232,11 +232,30 @@ export class QueryNodeApi { return this.getDataObjectsByBagIds(bagIds) } - public getActiveStorageBucketOperatorsData(): Promise { - return this.multipleEntitiesQuery< + public async getOperationallyActiveStorageBucketOperatorsData(): Promise { + const buckets = await this.multipleEntitiesQuery< GetActiveStorageBucketOperatorsDataQuery, GetActiveStorageBucketOperatorsDataQueryVariables >(GetActiveStorageBucketOperatorsData, {}, 'storageBuckets') + + // Filter out nodes/operators under maintenance + return buckets.filter(({ operatorMetadata }) => { + const status = operatorMetadata?.nodeOperationalStatus + const date = new Date() + if ( + !operatorMetadata || + !status || + status.__typename === 'NodeOperationalStatusNormal' || + (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet) + (status.__typename === 'NodeOperationalStatusNoServiceUntil' && + new Date(status.from) > date && + new Date(status.until) < date) // planned future maintenance with end time (which has not started yet) + ) { + return true + } + + return false + }) } public async getPackageVersion(): Promise { diff --git a/distributor-node/src/services/networking/query-node/queries/queries.graphql b/distributor-node/src/services/networking/query-node/queries/queries.graphql index 11297acf42..7a8e97a0cb 100644 --- a/distributor-node/src/services/networking/query-node/queries/queries.graphql +++ b/distributor-node/src/services/networking/query-node/queries/queries.graphql @@ -103,21 +103,31 @@ fragment StorageBucketOperatorFields on StorageBucket { id operatorMetadata { nodeEndpoint + nodeOperationalStatus { + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + from + } + ... on NodeOperationalStatusNoServiceUntil { + __typename + forced + from + until + } + } } } query getActiveStorageBucketOperatorsData { - storageBuckets( - where: { - operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } - operatorMetadata: { - OR: [ - { nodeOperationalStatus_isNull: true } - { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } - ] - } - } - ) { + storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) { ...StorageBucketOperatorFields } } diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql index 9a5be3c8a3..6991db6dcd 100644 --- a/distributor-node/src/services/networking/query-node/schema.graphql +++ b/distributor-node/src/services/networking/query-node/schema.graphql @@ -432,10 +432,10 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeOperationalStatus_from_DESC nodeOperationalStatus_from_ASC_NULLS_FIRST nodeOperationalStatus_from_DESC_NULLS_LAST - nodeOperationalStatus_to_ASC - nodeOperationalStatus_to_DESC - nodeOperationalStatus_to_ASC_NULLS_FIRST - nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST nodeOperationalStatus_isTypeOf_ASC nodeOperationalStatus_isTypeOf_DESC nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST @@ -678,7 +678,7 @@ input DistributionBucketWhereInput { type DistributionNodeOperationalStatusSetEvent { """Distribution bucket operator""" - bucketOperator: DistributionBucketOperator + bucketOperator: DistributionBucketOperator! """Operational status that was set""" operationalStatus: NodeOperationalStatus! @@ -1014,7 +1014,7 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } -union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceUntil type NodeOperationalStatusNormal { """Reason why node was set to this state""" @@ -1032,7 +1032,7 @@ type NodeOperationalStatusNoService { rationale: String } -type NodeOperationalStatusNoServiceDuring { +type NodeOperationalStatusNoServiceFrom { """ Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. @@ -1042,14 +1042,11 @@ type NodeOperationalStatusNoServiceDuring { """The time from which the bucket would have to no service""" from: DateTime! - """The time until which the bucket would have to no service""" - to: DateTime! - """Reason why node was set to this state""" rationale: String } -type NodeOperationalStatusNoServiceFrom { +type NodeOperationalStatusNoServiceUntil { """ Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. @@ -1059,6 +1056,9 @@ type NodeOperationalStatusNoServiceFrom { """The time from which the bucket would have to no service""" from: DateTime! + """The time until which the bucket would have to no service""" + until: DateTime! + """Reason why node was set to this state""" rationale: String } @@ -1093,15 +1093,15 @@ input NodeOperationalStatusWhereInput { from_lte: DateTime from_in: [DateTime!] from_not_in: [DateTime!] - to_isNull: Boolean - to_eq: DateTime - to_not_eq: DateTime - to_gt: DateTime - to_gte: DateTime - to_lt: DateTime - to_lte: DateTime - to_in: [DateTime!] - to_not_in: [DateTime!] + until_isNull: Boolean + until_eq: DateTime + until_not_eq: DateTime + until_gt: DateTime + until_gte: DateTime + until_lt: DateTime + until_lte: DateTime + until_in: [DateTime!] + until_not_in: [DateTime!] isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -1181,10 +1181,6 @@ type Query { distributionBucketFamilyById(id: String!): DistributionBucketFamily distributionBucketFamilyByUniqueInput(where: WhereIdInput!): DistributionBucketFamily @deprecated(reason: "Use distributionBucketFamilyById") distributionBucketFamiliesConnection(orderBy: [DistributionBucketFamilyOrderByInput!]!, after: String, first: Int, where: DistributionBucketFamilyWhereInput): DistributionBucketFamiliesConnection! - workers(where: WorkerWhereInput, orderBy: [WorkerOrderByInput!], offset: Int, limit: Int): [Worker!]! - workerById(id: String!): Worker - workerByUniqueInput(where: WhereIdInput!): Worker @deprecated(reason: "Use workerById") - workersConnection(orderBy: [WorkerOrderByInput!]!, after: String, first: Int, where: WorkerWhereInput): WorkersConnection! squidStatus: SquidStatus squidVersion: SquidVersion! } @@ -1597,10 +1593,10 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeOperationalStatus_from_DESC nodeOperationalStatus_from_ASC_NULLS_FIRST nodeOperationalStatus_from_DESC_NULLS_LAST - nodeOperationalStatus_to_ASC - nodeOperationalStatus_to_DESC - nodeOperationalStatus_to_ASC_NULLS_FIRST - nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST nodeOperationalStatus_isTypeOf_ASC nodeOperationalStatus_isTypeOf_DESC nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST @@ -2149,63 +2145,3 @@ input WhereIdInput { id: String! } -type Worker { - """Worker id ({workingGroupName}-{workerId})""" - id: String! - - """WorkerId in specific working group module""" - runtimeId: BigInt! -} - -type WorkerEdge { - node: Worker! - cursor: String! -} - -enum WorkerOrderByInput { - id_ASC - id_DESC - id_ASC_NULLS_FIRST - id_DESC_NULLS_LAST - runtimeId_ASC - runtimeId_DESC - runtimeId_ASC_NULLS_FIRST - runtimeId_DESC_NULLS_LAST -} - -type WorkersConnection { - edges: [WorkerEdge!]! - pageInfo: PageInfo! - totalCount: Int! -} - -input WorkerWhereInput { - id_isNull: Boolean - id_eq: String - id_not_eq: String - id_gt: String - id_gte: String - id_lt: String - id_lte: String - id_in: [String!] - id_not_in: [String!] - id_contains: String - id_not_contains: String - id_containsInsensitive: String - id_not_containsInsensitive: String - id_startsWith: String - id_not_startsWith: String - id_endsWith: String - id_not_endsWith: String - runtimeId_isNull: Boolean - runtimeId_eq: BigInt - runtimeId_not_eq: BigInt - runtimeId_gt: BigInt - runtimeId_gte: BigInt - runtimeId_lt: BigInt - runtimeId_lte: BigInt - runtimeId_in: [BigInt!] - runtimeId_not_in: [BigInt!] - AND: [WorkerWhereInput!] - OR: [WorkerWhereInput!] -} diff --git a/storage-node/src/services/queryNode/api.ts b/storage-node/src/services/queryNode/api.ts index 358c17db8a..31fd2a026b 100644 --- a/storage-node/src/services/queryNode/api.ts +++ b/storage-node/src/services/queryNode/api.ts @@ -182,6 +182,29 @@ export class QueryNodeApi { return result.data[resultKey] } + private getOperationallyActiveStorageBuckets( + buckets: Array + ): Array { + // Filter out nodes/operators under maintenance + return buckets.filter(({ operatorMetadata }) => { + const status = operatorMetadata?.nodeOperationalStatus + const date = new Date() + if ( + !operatorMetadata || + !status || + status.__typename === 'NodeOperationalStatusNormal' || + (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet) + (status.__typename === 'NodeOperationalStatusNoServiceUntil' && + new Date(status.from) > date && + new Date(status.until) < date) // planned future maintenance with end time (which has not started yet) + ) { + return true + } + + return false + }) + } + /** * Returns storage bucket IDs filtered by worker ID. * @@ -197,7 +220,7 @@ export class QueryNodeApi { return [] } - return result + return this.getOperationallyActiveStorageBuckets(result) } /** @@ -301,7 +324,7 @@ export class QueryNodeApi { return [] } - return result + return this.getOperationallyActiveStorageBuckets(result) } /** diff --git a/storage-node/src/services/queryNode/queries/queries.graphql b/storage-node/src/services/queryNode/queries/queries.graphql index b7ceed17b7..bf6930de4a 100644 --- a/storage-node/src/services/queryNode/queries/queries.graphql +++ b/storage-node/src/services/queryNode/queries/queries.graphql @@ -1,34 +1,43 @@ +fragment NodeOperationalStatusFragment on NodeOperationalStatus { + __typename + ... on NodeOperationalStatusNormal { + __typename + } + ... on NodeOperationalStatusNoService { + __typename + forced + } + ... on NodeOperationalStatusNoServiceFrom { + __typename + forced + from + } + ... on NodeOperationalStatusNoServiceUntil { + __typename + forced + from + until + } +} + fragment StorageBucketIds on StorageBucket { id + operatorMetadata { + nodeOperationalStatus { + ...NodeOperationalStatusFragment + } + } } query getStorageBuckets { - storageBuckets( - where: { - operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } - operatorMetadata: { - OR: [ - { nodeOperationalStatus_isNull: true } - { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } - ] - } - } - ) { + storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) { ...StorageBucketIds } } query getStorageBucketsByWorkerId($workerId: BigInt!) { storageBuckets( - where: { - operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } - operatorMetadata: { - OR: [ - { nodeOperationalStatus_isNull: true } - { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } } - ] - } - } + where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } } ) { ...StorageBucketIds } @@ -49,10 +58,13 @@ query getStorageBucketsOperationalStatus($ids: [String!]) { ... on NodeOperationalStatusNoServiceFrom { __typename forced + from } - ... on NodeOperationalStatusNoServiceDuring { + ... on NodeOperationalStatusNoServiceUntil { __typename forced + from + until } } } diff --git a/storage-node/src/services/queryNode/schema.graphql b/storage-node/src/services/queryNode/schema.graphql index 9a5be3c8a3..6991db6dcd 100644 --- a/storage-node/src/services/queryNode/schema.graphql +++ b/storage-node/src/services/queryNode/schema.graphql @@ -432,10 +432,10 @@ enum DistributionBucketOperatorMetadataOrderByInput { nodeOperationalStatus_from_DESC nodeOperationalStatus_from_ASC_NULLS_FIRST nodeOperationalStatus_from_DESC_NULLS_LAST - nodeOperationalStatus_to_ASC - nodeOperationalStatus_to_DESC - nodeOperationalStatus_to_ASC_NULLS_FIRST - nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST nodeOperationalStatus_isTypeOf_ASC nodeOperationalStatus_isTypeOf_DESC nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST @@ -678,7 +678,7 @@ input DistributionBucketWhereInput { type DistributionNodeOperationalStatusSetEvent { """Distribution bucket operator""" - bucketOperator: DistributionBucketOperator + bucketOperator: DistributionBucketOperator! """Operational status that was set""" operationalStatus: NodeOperationalStatus! @@ -1014,7 +1014,7 @@ input NodeLocationMetadataWhereInput { coordinates: GeoCoordinatesWhereInput } -union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring +union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceUntil type NodeOperationalStatusNormal { """Reason why node was set to this state""" @@ -1032,7 +1032,7 @@ type NodeOperationalStatusNoService { rationale: String } -type NodeOperationalStatusNoServiceDuring { +type NodeOperationalStatusNoServiceFrom { """ Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. @@ -1042,14 +1042,11 @@ type NodeOperationalStatusNoServiceDuring { """The time from which the bucket would have to no service""" from: DateTime! - """The time until which the bucket would have to no service""" - to: DateTime! - """Reason why node was set to this state""" rationale: String } -type NodeOperationalStatusNoServiceFrom { +type NodeOperationalStatusNoServiceUntil { """ Whether the state was set by lead (true) or by the operator (false), it is meant to prevent worker from unilaterally reversing. @@ -1059,6 +1056,9 @@ type NodeOperationalStatusNoServiceFrom { """The time from which the bucket would have to no service""" from: DateTime! + """The time until which the bucket would have to no service""" + until: DateTime! + """Reason why node was set to this state""" rationale: String } @@ -1093,15 +1093,15 @@ input NodeOperationalStatusWhereInput { from_lte: DateTime from_in: [DateTime!] from_not_in: [DateTime!] - to_isNull: Boolean - to_eq: DateTime - to_not_eq: DateTime - to_gt: DateTime - to_gte: DateTime - to_lt: DateTime - to_lte: DateTime - to_in: [DateTime!] - to_not_in: [DateTime!] + until_isNull: Boolean + until_eq: DateTime + until_not_eq: DateTime + until_gt: DateTime + until_gte: DateTime + until_lt: DateTime + until_lte: DateTime + until_in: [DateTime!] + until_not_in: [DateTime!] isTypeOf_isNull: Boolean isTypeOf_eq: String isTypeOf_not_eq: String @@ -1181,10 +1181,6 @@ type Query { distributionBucketFamilyById(id: String!): DistributionBucketFamily distributionBucketFamilyByUniqueInput(where: WhereIdInput!): DistributionBucketFamily @deprecated(reason: "Use distributionBucketFamilyById") distributionBucketFamiliesConnection(orderBy: [DistributionBucketFamilyOrderByInput!]!, after: String, first: Int, where: DistributionBucketFamilyWhereInput): DistributionBucketFamiliesConnection! - workers(where: WorkerWhereInput, orderBy: [WorkerOrderByInput!], offset: Int, limit: Int): [Worker!]! - workerById(id: String!): Worker - workerByUniqueInput(where: WhereIdInput!): Worker @deprecated(reason: "Use workerById") - workersConnection(orderBy: [WorkerOrderByInput!]!, after: String, first: Int, where: WorkerWhereInput): WorkersConnection! squidStatus: SquidStatus squidVersion: SquidVersion! } @@ -1597,10 +1593,10 @@ enum StorageBucketOperatorMetadataOrderByInput { nodeOperationalStatus_from_DESC nodeOperationalStatus_from_ASC_NULLS_FIRST nodeOperationalStatus_from_DESC_NULLS_LAST - nodeOperationalStatus_to_ASC - nodeOperationalStatus_to_DESC - nodeOperationalStatus_to_ASC_NULLS_FIRST - nodeOperationalStatus_to_DESC_NULLS_LAST + nodeOperationalStatus_until_ASC + nodeOperationalStatus_until_DESC + nodeOperationalStatus_until_ASC_NULLS_FIRST + nodeOperationalStatus_until_DESC_NULLS_LAST nodeOperationalStatus_isTypeOf_ASC nodeOperationalStatus_isTypeOf_DESC nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST @@ -2149,63 +2145,3 @@ input WhereIdInput { id: String! } -type Worker { - """Worker id ({workingGroupName}-{workerId})""" - id: String! - - """WorkerId in specific working group module""" - runtimeId: BigInt! -} - -type WorkerEdge { - node: Worker! - cursor: String! -} - -enum WorkerOrderByInput { - id_ASC - id_DESC - id_ASC_NULLS_FIRST - id_DESC_NULLS_LAST - runtimeId_ASC - runtimeId_DESC - runtimeId_ASC_NULLS_FIRST - runtimeId_DESC_NULLS_LAST -} - -type WorkersConnection { - edges: [WorkerEdge!]! - pageInfo: PageInfo! - totalCount: Int! -} - -input WorkerWhereInput { - id_isNull: Boolean - id_eq: String - id_not_eq: String - id_gt: String - id_gte: String - id_lt: String - id_lte: String - id_in: [String!] - id_not_in: [String!] - id_contains: String - id_not_contains: String - id_containsInsensitive: String - id_not_containsInsensitive: String - id_startsWith: String - id_not_startsWith: String - id_endsWith: String - id_not_endsWith: String - runtimeId_isNull: Boolean - runtimeId_eq: BigInt - runtimeId_not_eq: BigInt - runtimeId_gt: BigInt - runtimeId_gte: BigInt - runtimeId_lt: BigInt - runtimeId_lte: BigInt - runtimeId_in: [BigInt!] - runtimeId_not_in: [BigInt!] - AND: [WorkerWhereInput!] - OR: [WorkerWhereInput!] -}