From 2c9c653dfde51127f3ff2c29845a0d94cef04fb4 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 18:00:35 +0530 Subject: [PATCH 01/48] push initial setup --- Dockerfile | 94 +++++++++++++++++++ Jenkinsfile | 87 +++++++++++++++++ docker/hyperdx/entry.prod.sh | 18 +++- .../src/components/DBSearchPageFilters.tsx | 74 ++++++++------- 4 files changed, 237 insertions(+), 36 deletions(-) create mode 100644 Dockerfile create mode 100644 Jenkinsfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..019e3b8eb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,94 @@ +# HyperDX API and App Server Dockerfile +# This Dockerfile creates a single image with both the API and App servers + +ARG NODE_VERSION=22.16.0 + +# Base stage with Node.js and dependencies +FROM node:${NODE_VERSION}-alpine AS base + +WORKDIR /app + +# Copy workspace configuration files +COPY .yarn ./.yarn +COPY .yarnrc.yml yarn.lock package.json nx.json .prettierrc .prettierignore ./ + +# Copy package.json files for all packages +COPY ./packages/common-utils/package.json ./packages/common-utils/ +COPY ./packages/api/package.json ./packages/api/ +COPY ./packages/app/package.json ./packages/app/ + +# Install dependencies +RUN apk add --no-cache libc6-compat +RUN yarn install --mode=skip-build && yarn cache clean + +# Builder stage +FROM base AS builder + +WORKDIR /app + +# Copy source code for all packages +COPY ./packages/common-utils ./packages/common-utils +COPY ./packages/api ./packages/api +COPY ./packages/app ./packages/app + +# Set build environment variables +ENV NEXT_TELEMETRY_DISABLED 1 +ENV NEXT_PUBLIC_IS_LOCAL_MODE false +ENV NX_DAEMON=false + +# Build packages in dependency order +RUN yarn workspace @hyperdx/common-utils build +RUN yarn workspace @hyperdx/api build +RUN yarn workspace @hyperdx/app build + +# Production stage +FROM node:${NODE_VERSION}-alpine AS production + +ARG CODE_VERSION=2.1.1 + +ENV CODE_VERSION=2.1.1 +ENV NODE_ENV production + +# Install concurrently for running multiple processes +RUN npm install -g concurrently@9.1.0 + +# Create non-root user +RUN addgroup -g 1001 -S nodejs +RUN adduser -S nodejs -u 1001 + +USER nodejs + +WORKDIR /app + +# Copy built API +COPY --chown=nodejs:nodejs --from=builder /app/packages/api/dist ./packages/api/dist +COPY --chown=nodejs:nodejs --from=builder /app/packages/api/package.json ./packages/api/package.json + +# Copy built App (Next.js) +COPY --chown=nodejs:nodejs --from=builder /app/packages/app/.next ./packages/app/.next +COPY --chown=nodejs:nodejs --from=builder /app/packages/app/public ./packages/app/public +COPY --chown=nodejs:nodejs --from=builder /app/packages/app/package.json ./packages/app/package.json +COPY --chown=nodejs:nodejs --from=builder /app/packages/app/next.config.js ./packages/app/next.config.js + +# Copy built common-utils +COPY --chown=nodejs:nodejs --from=builder /app/packages/common-utils/dist ./packages/common-utils/dist +COPY --chown=nodejs:nodejs --from=builder /app/packages/common-utils/package.json ./packages/common-utils/package.json + +# Copy node_modules for runtime dependencies +COPY --chown=nodejs:nodejs --from=builder /app/node_modules ./node_modules +COPY --chown=nodejs:nodejs --from=builder /app/packages/api/node_modules ./packages/api/node_modules +COPY --chown=nodejs:nodejs --from=builder /app/packages/app/node_modules ./packages/app/node_modules +COPY --chown=nodejs:nodejs --from=builder /app/packages/common-utils/node_modules ./packages/common-utils/node_modules + +# Copy and set up entry script +COPY --chown=nodejs:nodejs docker/hyperdx/entry.prod.sh /etc/local/entry.sh +RUN chmod +x /etc/local/entry.sh + +# Expose ports +EXPOSE 8000 8080 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 + +ENTRYPOINT ["sh", "/etc/local/entry.sh"] \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..959611225 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,87 @@ +pipeline { + agent { + kubernetes { + label 'hyperdx' + yaml """ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: dind + image: sc-mum-armory.platform.internal/devops/dind:v2 + securityContext: + privileged: true + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: DOCKER_TLS_CERTDIR + value: "" + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + readinessProbe: + tcpSocket: + port: 2375 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 2375 + initialDelaySeconds: 30 + periodSeconds: 20 + - name: builder + image: sc-mum-armory.platform.internal/devops/builder-image-armory + command: + - sleep + - infinity + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: DOCKER_BUILDKIT + value: "0" + volumeMounts: + - name: jenkins-sa + mountPath: /root/.gcp/ + volumes: + - name: dind-storage + emptyDir: {} + - name: jenkins-sa + secret: + secretName: jenkins-sa +""" + } + } + + environment { + sc_regions="mumbai" + GITHUB_TOKEN = credentials('github-access') + app="hyperdx" + buildarg_DEPLOYMENT_ID="hyperdx-$GIT_COMMIT" + buildarg_GITHUB_TOKEN="${GITHUB_TOKEN}" + } + + stages { + stage('build') { + steps { + container('builder') { + sh "armory build" + } + } + } + + stage('push') { + when { + anyOf { + branch 'main' + branch 'staging' + branch 'feature/*' + } + } + steps { + container('builder') { + sh "armory push" + } + } + } + } +} \ No newline at end of file diff --git a/docker/hyperdx/entry.prod.sh b/docker/hyperdx/entry.prod.sh index cf619611d..88a0d2283 100644 --- a/docker/hyperdx/entry.prod.sh +++ b/docker/hyperdx/entry.prod.sh @@ -8,13 +8,25 @@ export OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320} export IS_LOCAL_APP_MODE="REQUIRED_AUTH" echo "" +echo "Starting HyperDX services..." echo "Visit the HyperDX UI at $FRONTEND_URL" echo "" +# Check if required files exist +if [ ! -f "./packages/api/dist/index.js" ]; then + echo "ERROR: API build not found at ./packages/api/dist/index.js" + exit 1 +fi + +if [ ! -d "./packages/app/.next" ]; then + echo "ERROR: App build not found at ./packages/app/.next" + exit 1 +fi + # Use concurrently to run both the API and App servers npx concurrently \ "--kill-others-on-fail" \ "--names=API,APP,ALERT-TASK" \ - "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./packages/api/tracing ./packages/api/index" \ - "cd ./packages/app/packages/app && HOSTNAME='0.0.0.0' HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} PORT=${HYPERDX_APP_PORT:-8080} node server.js" \ - "node -r ./packages/api/tracing ./packages/api/tasks/index check-alerts" + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./packages/api/dist/tracing.js ./packages/api/dist/index.js" \ + "cd ./packages/app && HOSTNAME='0.0.0.0' HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} PORT=${HYPERDX_APP_PORT:-8080} yarn start" \ + "node -r ./packages/api/dist/tracing.js ./packages/api/dist/tasks/index.js check-alerts" diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 6fe7ff027..423ad9226 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -29,6 +29,37 @@ import { mergePath } from '@/utils'; import resizeStyles from '../../styles/ResizablePanel.module.scss'; import classes from '../../styles/SearchPage.module.scss'; +// Override keys for specific source types +const serviceMapOverride = { + Logs: [ + 'SeverityText', + 'ServiceName', + "ResourceAttributes['k8s.cluster.name']", + "ResourceAttributes['k8s.namespace.name']", + ], + Traces: [ + 'ServiceName', + 'StatusCode', + "ResourceAttributes['k8s.node.name']", + "ResourceAttributes['k8s.owner.name']", + 'SpanKind', + ], + 'K8s Events': [ + "ResourceAttributes['k8s.cluster.name']", + "LogAttributes['k8s.namespace.name']", + "LogAttributes['k8s.event.reason']", + ], +}; + +// Helper function to get keys - override for specific types, use default for others +const getKeysForSourceType = (sourceType?: string) => { + if (sourceType && sourceType in serviceMapOverride) { + return serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; + } + // For other source types, return empty array to use default behavior + return []; +}; + type FilterCheckboxProps = { label: string; value?: 'included' | 'excluded' | false; @@ -415,39 +446,15 @@ const DBSearchPageFiltersComponent = ({ const [showMoreFields, setShowMoreFields] = useState(false); const keysToFetch = useMemo(() => { - if (!data) { - return []; + // Override keys for specific source types + if (sourceType && sourceType in serviceMapOverride) { + return getKeysForSourceType(sourceType); } - - const strings = data - .sort((a, b) => { - // First show low cardinality fields - const isLowCardinality = (type: string) => - type.includes('LowCardinality'); - return isLowCardinality(a.type) && !isLowCardinality(b.type) ? -1 : 1; - }) - .filter( - field => field.jsType && ['string'].includes(field.jsType), - // todo: add number type with sliders :D - ) - .map(({ path, type }) => { - return { type, path: mergePath(path) }; - }) - .filter( - field => - showMoreFields || - field.type.includes('LowCardinality') || // query only low cardinality fields by default - Object.keys(filterState).includes(field.path) || // keep selected fields - isFieldPinned(field.path), // keep pinned fields - ) - .map(({ path }) => path) - .filter( - path => - !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()), - ); - - return strings; - }, [data, filterState, showMoreFields]); + console.error('nishant is here === ', sourceType); + // For other source types, return empty array to use default behavior + // This allows the system to fetch all available keys from metadata + return []; + }, [sourceType]); // Special case for live tail const [dateRange, setDateRange] = useState<[Date, Date]>( @@ -471,10 +478,11 @@ const DBSearchPageFiltersComponent = ({ } = useGetKeyValues({ chartConfigs: { ...chartConfig, dateRange }, limit: keyLimit, - keys: keysToFetch, + keys: getKeysForSourceType(sourceType), }); const [extraFacets, setExtraFacets] = useState>({}); + const [loadMoreLoadingKeys, setLoadMoreLoadingKeys] = useState>( new Set(), ); From 9a6ac0a62e66c0a3275a66c75e5818b51b0c1226 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 18:17:32 +0530 Subject: [PATCH 02/48] update --- packages/app/src/components/DBSearchPageFilters.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 423ad9226..91c36c0c5 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -197,12 +197,17 @@ export const FilterGroup = ({ onFieldPinClick, isFieldPinned, onLoadMore, - loadMoreLoading, hasLoadedMore, }: FilterGroupProps) => { const [search, setSearch] = useState(''); const [isExpanded, setExpanded] = useState(false); + useEffect(() => { + if (onLoadMore && !hasLoadedMore) { + onLoadMore(name); + } + }, [onLoadMore, hasLoadedMore, name]); + const augmentedOptions = useMemo(() => { const selectedSet = new Set([ ...selectedValues.included, @@ -412,6 +417,7 @@ const DBSearchPageFiltersComponent = ({ analysisMode, setAnalysisMode, sourceId, + sourceType, showDelta, denoiseResults, setDenoiseResults, @@ -517,11 +523,13 @@ const DBSearchPageFiltersComponent = ({ }); } }, - [chartConfig, setExtraFacets, dateRange], + [chartConfig, setExtraFacets, dateRange, sourceType], ); const shownFacets = useMemo(() => { const _facets: { key: string; value: string[] }[] = []; + + //for filters for (const facet of facets ?? []) { // don't include empty facets, unless they are already selected const filter = filterState[facet.key]; From 0a3047a72598975d90932fc355319f74b2fa45e6 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 18:26:47 +0530 Subject: [PATCH 03/48] update --- packages/app/src/components/DBSearchPageFilters.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 91c36c0c5..2ee3bd2d4 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -197,6 +197,7 @@ export const FilterGroup = ({ onFieldPinClick, isFieldPinned, onLoadMore, + loadMoreLoading, hasLoadedMore, }: FilterGroupProps) => { const [search, setSearch] = useState(''); @@ -427,6 +428,7 @@ const DBSearchPageFiltersComponent = ({ isLive: boolean; chartConfig: ChartConfigWithDateRange; sourceId?: string; + sourceType?: string; showDelta: boolean; denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; From f719fe236944c981b60cd2a76cacf0eeab9f91e7 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 18:44:34 +0530 Subject: [PATCH 04/48] update --- packages/app/src/DBSearchPage.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 40c1c2e8b..6562b597a 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -1417,6 +1417,7 @@ function DBSearchPage() { setAnalysisMode={setAnalysisMode} chartConfig={filtersChartConfig} sourceId={inputSourceObj?.id} + sourceType={inputSourceObj?.kind} showDelta={!!searchedSource?.durationExpression} {...searchFilters} /> From f9b9fd59fe686281e09144928a6ef1927452bcad Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 20:05:40 +0530 Subject: [PATCH 05/48] update --- .../src/components/DBSearchPageFilters.tsx | 104 +++++++++++++----- 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 2ee3bd2d4..ef632b805 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -29,27 +29,32 @@ import { mergePath } from '@/utils'; import resizeStyles from '../../styles/ResizablePanel.module.scss'; import classes from '../../styles/SearchPage.module.scss'; -// Override keys for specific source types -const serviceMapOverride = { - Logs: [ - 'SeverityText', - 'ServiceName', - "ResourceAttributes['k8s.cluster.name']", - "ResourceAttributes['k8s.namespace.name']", - ], - Traces: [ - 'ServiceName', - 'StatusCode', - "ResourceAttributes['k8s.node.name']", - "ResourceAttributes['k8s.owner.name']", - 'SpanKind', - ], - 'K8s Events': [ - "ResourceAttributes['k8s.cluster.name']", - "LogAttributes['k8s.namespace.name']", - "LogAttributes['k8s.event.reason']", - ], -}; + // Override keys for specific source types + const serviceMapOverride = { + log: [ + 'SeverityText', + 'ServiceName', + "ResourceAttributes['k8s.cluster.name']", + "ResourceAttributes['k8s.namespace.name']", + ], + trace: [ + 'ServiceName', + 'StatusCode', + "ResourceAttributes['k8s.node.name']", + "ResourceAttributes['k8s.owner.name']", + 'SpanKind', + ], + session: [ + 'ServiceName', + 'StatusCode', + 'SpanKind', + ], + metric: [ + 'ServiceName', + 'MetricName', + 'Unit', + ], + }; // Helper function to get keys - override for specific types, use default for others const getKeysForSourceType = (sourceType?: string) => { @@ -453,16 +458,57 @@ const DBSearchPageFiltersComponent = ({ const [showMoreFields, setShowMoreFields] = useState(false); - const keysToFetch = useMemo(() => { - // Override keys for specific source types + // const keysToFetch = useMemo(() => { + // // Get keys for the source type - will return specific keys if available, or default keys otherwise + // return getKeysForSourceType(sourceType); + // }, [sourceType]); + + const keysToFetch = useMemo(() => { + console.log('🔍 keysToFetch - sourceType:', sourceType, 'available overrides:', Object.keys(serviceMapOverride)); + + // First check if we have a source type override if (sourceType && sourceType in serviceMapOverride) { - return getKeysForSourceType(sourceType); + const overrideKeys = serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; + console.log('✅ Using source type override for', sourceType, ':', overrideKeys); + return overrideKeys; + } + + console.log('⚠️ No source type override found for', sourceType, '- falling back to data-based logic'); + + // If no source type override, fall back to data-based logic + if (!data) { + return []; } - console.error('nishant is here === ', sourceType); - // For other source types, return empty array to use default behavior - // This allows the system to fetch all available keys from metadata - return []; - }, [sourceType]); + + const strings = data + .sort((a, b) => { + // First show low cardinality fields + const isLowCardinality = (type: string) => + type.includes('LowCardinality'); + return isLowCardinality(a.type) && !isLowCardinality(b.type) ? -1 : 1; + }) + .filter( + field => field.jsType && ['string'].includes(field.jsType), + // todo: add number type with sliders :D + ) + .map(({ path, type }) => { + return { type, path: mergePath(path) }; + }) + .filter( + field => + showMoreFields || + field.type.includes('LowCardinality') || // query only low cardinality fields by default + Object.keys(filterState).includes(field.path) || // keep selected fields + isFieldPinned(field.path), // keep pinned fields + ) + .map(({ path }) => path) + .filter( + path => + !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()), + ); + + return strings; + }, [sourceType, data, filterState, showMoreFields]); // Special case for live tail const [dateRange, setDateRange] = useState<[Date, Date]>( From bffab63926fee29b58693c912d7daab162c1f703 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 20:31:46 +0530 Subject: [PATCH 06/48] update --- .../src/components/DBSearchPageFilters.tsx | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index ef632b805..d05bb89d4 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -29,32 +29,24 @@ import { mergePath } from '@/utils'; import resizeStyles from '../../styles/ResizablePanel.module.scss'; import classes from '../../styles/SearchPage.module.scss'; - // Override keys for specific source types - const serviceMapOverride = { - log: [ - 'SeverityText', - 'ServiceName', - "ResourceAttributes['k8s.cluster.name']", - "ResourceAttributes['k8s.namespace.name']", - ], - trace: [ - 'ServiceName', - 'StatusCode', - "ResourceAttributes['k8s.node.name']", - "ResourceAttributes['k8s.owner.name']", - 'SpanKind', - ], - session: [ - 'ServiceName', - 'StatusCode', - 'SpanKind', - ], - metric: [ - 'ServiceName', - 'MetricName', - 'Unit', - ], - }; +// Override keys for specific source types +const serviceMapOverride = { + log: [ + 'SeverityText', + 'ServiceName', + "ResourceAttributes['k8s.cluster.name']", + "ResourceAttributes['k8s.namespace.name']", + ], + trace: [ + 'ServiceName', + 'StatusCode', + "ResourceAttributes['k8s.node.name']", + "ResourceAttributes['k8s.owner.name']", + 'SpanKind', + ], + session: ['ServiceName', 'StatusCode', 'SpanKind'], + metric: ['ServiceName', 'MetricName', 'Unit'], +}; // Helper function to get keys - override for specific types, use default for others const getKeysForSourceType = (sourceType?: string) => { @@ -99,7 +91,9 @@ export const TextButton = ({ ); }; -const emptyFn = () => {}; +const emptyFn = () => { + // Intentionally empty - onChange is handled by parent Group's onClick +}; export const FilterCheckbox = ({ value, label, @@ -279,7 +273,7 @@ export const FilterGroup = ({ selectedValues.included.size + selectedValues.excluded.size, ), ); - }, [search, isExpanded, augmentedOptions, selectedValues]); + }, [search, isExpanded, augmentedOptions, selectedValues, isPinned]); const showExpandButton = !search && @@ -438,17 +432,11 @@ const DBSearchPageFiltersComponent = ({ denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; } & FilterStateHook) => { - const { - toggleFilterPin, - toggleFieldPin, - isFilterPinned, - isFieldPinned, - getPinnedFields, - } = usePinnedFilters(sourceId ?? null); + const { toggleFilterPin, toggleFieldPin, isFilterPinned, isFieldPinned } = + usePinnedFilters(sourceId ?? null); const { width, startResize } = useResizable(16, 'left'); const { data: countData } = useExplainQuery(chartConfig); - const numRows: number = countData?.[0]?.rows ?? 0; const { data, isLoading } = useAllFields({ databaseName: chartConfig.from.databaseName, @@ -463,12 +451,17 @@ const DBSearchPageFiltersComponent = ({ // return getKeysForSourceType(sourceType); // }, [sourceType]); - const keysToFetch = useMemo(() => { - console.log('🔍 keysToFetch - sourceType:', sourceType, 'available overrides:', Object.keys(serviceMapOverride)); + const keysToFetch = useMemo(() => { + console.log('🔍 keysToFetch function called with:'); + console.log(' - sourceType:', sourceType); + console.log(' - data available:', !!data); + console.log(' - data length:', data?.length || 0); + console.log(' - serviceMapOverride keys:', Object.keys(serviceMapOverride)); // First check if we have a source type override if (sourceType && sourceType in serviceMapOverride) { - const overrideKeys = serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; + const overrideKeys = + serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; console.log('✅ Using source type override for', sourceType, ':', overrideKeys); return overrideKeys; } @@ -477,8 +470,13 @@ const DBSearchPageFiltersComponent = ({ // If no source type override, fall back to data-based logic if (!data) { + console.log('❌ No data available, returning empty array'); return []; } + console.log('📊 Processing data for keysToFetch:'); + console.log(' - Raw data sample:', data.slice(0, 3)); + console.log(' - showMoreFields:', showMoreFields); + console.log(' - filterState keys:', Object.keys(filterState)); const strings = data .sort((a, b) => { @@ -507,6 +505,10 @@ const DBSearchPageFiltersComponent = ({ !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()), ); + console.log('🎯 Final keysToFetch result:', strings); + console.log(' - Total keys found:', strings.length); + console.log(' - Keys:', strings); + return strings; }, [sourceType, data, filterState, showMoreFields]); From 802a4dbc6231a7abd446db9aa8e184ef78fc63d6 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 20:43:18 +0530 Subject: [PATCH 07/48] update --- .../src/components/DBSearchPageFilters.tsx | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index d05bb89d4..f83bef2c4 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -451,32 +451,32 @@ const DBSearchPageFiltersComponent = ({ // return getKeysForSourceType(sourceType); // }, [sourceType]); - const keysToFetch = useMemo(() => { - console.log('🔍 keysToFetch function called with:'); - console.log(' - sourceType:', sourceType); - console.log(' - data available:', !!data); - console.log(' - data length:', data?.length || 0); - console.log(' - serviceMapOverride keys:', Object.keys(serviceMapOverride)); + const keysToFetch = useMemo(() => { + console.error('🔍 keysToFetch function called with:'); + console.error(' - sourceType:', sourceType); + console.error(' - data available:', !!data); + console.error(' - data length:', data?.length || 0); + console.error(' - serviceMapOverride keys:', Object.keys(serviceMapOverride)); // First check if we have a source type override if (sourceType && sourceType in serviceMapOverride) { const overrideKeys = serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; - console.log('✅ Using source type override for', sourceType, ':', overrideKeys); + console.error('✅ Using source type override for', sourceType, ':', overrideKeys); return overrideKeys; } - console.log('⚠️ No source type override found for', sourceType, '- falling back to data-based logic'); + console.error('⚠️ No source type override found for', sourceType, '- falling back to data-based erroric'); - // If no source type override, fall back to data-based logic + // If no source type override, fall back to data-based erroric if (!data) { - console.log('❌ No data available, returning empty array'); + console.error('❌ No data available, returning empty array'); return []; } - console.log('📊 Processing data for keysToFetch:'); - console.log(' - Raw data sample:', data.slice(0, 3)); - console.log(' - showMoreFields:', showMoreFields); - console.log(' - filterState keys:', Object.keys(filterState)); + console.error('📊 Processing data for keysToFetch:'); + console.error(' - Raw data sample:', data.slice(0, 3)); + console.error(' - showMoreFields:', showMoreFields); + console.error(' - filterState keys:', Object.keys(filterState)); const strings = data .sort((a, b) => { @@ -505,9 +505,9 @@ const DBSearchPageFiltersComponent = ({ !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()), ); - console.log('🎯 Final keysToFetch result:', strings); - console.log(' - Total keys found:', strings.length); - console.log(' - Keys:', strings); + console.error('🎯 Final keysToFetch result:', strings); + console.error(' - Total keys found:', strings.length); + console.error(' - Keys:', strings); return strings; }, [sourceType, data, filterState, showMoreFields]); From 352d9c81e12e10d61e935b07de8c5ea05ff22c16 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 20:50:46 +0530 Subject: [PATCH 08/48] update --- .../src/components/DBSearchPageFilters.tsx | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index f83bef2c4..3d89ea3d8 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -456,19 +456,31 @@ const DBSearchPageFiltersComponent = ({ console.error(' - sourceType:', sourceType); console.error(' - data available:', !!data); console.error(' - data length:', data?.length || 0); - console.error(' - serviceMapOverride keys:', Object.keys(serviceMapOverride)); + console.error( + ' - serviceMapOverride keys:', + Object.keys(serviceMapOverride), + ); // First check if we have a source type override if (sourceType && sourceType in serviceMapOverride) { const overrideKeys = serviceMapOverride[sourceType as keyof typeof serviceMapOverride]; - console.error('✅ Using source type override for', sourceType, ':', overrideKeys); + console.error( + '✅ Using source type override for', + sourceType, + ':', + overrideKeys, + ); return overrideKeys; } - console.error('⚠️ No source type override found for', sourceType, '- falling back to data-based erroric'); + console.error( + '⚠️ No source type override found for', + sourceType, + '- falling back to data-based logic', + ); - // If no source type override, fall back to data-based erroric + // If no source type override, fall back to data-based logic if (!data) { console.error('❌ No data available, returning empty array'); return []; From 6827171e3897c3a483755798c2ce9a4fb4f390e1 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 21:29:42 +0530 Subject: [PATCH 09/48] update --- packages/app/src/components/DBSearchPageFilters.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 3d89ea3d8..02108e015 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -469,7 +469,7 @@ const DBSearchPageFiltersComponent = ({ '✅ Using source type override for', sourceType, ':', - overrideKeys, + DBSearchPageFiltersComponent, ); return overrideKeys; } From 37d36df39a8e07686308cebc5665d5984b98ca96 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 21:30:34 +0530 Subject: [PATCH 10/48] update --- packages/app/src/components/DBSearchPageFilters.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 02108e015..13841a500 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -469,7 +469,7 @@ const DBSearchPageFiltersComponent = ({ '✅ Using source type override for', sourceType, ':', - DBSearchPageFiltersComponent, + overrideKeys, ); return overrideKeys; } @@ -488,7 +488,7 @@ const DBSearchPageFiltersComponent = ({ console.error('📊 Processing data for keysToFetch:'); console.error(' - Raw data sample:', data.slice(0, 3)); console.error(' - showMoreFields:', showMoreFields); - console.error(' - filterState keys:', Object.keys(filterState)); + console.error(' - filterState keys:', Object.keys(DBSearchPageFiltersComponent)); const strings = data .sort((a, b) => { From 940eef7fadc60f628abf8d8943aa633cadf29a5f Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 23:45:10 +0530 Subject: [PATCH 11/48] update --- .../src/components/DBSearchPageFilters.tsx | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 13841a500..30e5234ae 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -432,9 +432,34 @@ const DBSearchPageFiltersComponent = ({ denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; } & FilterStateHook) => { + // Log all component props and variables + console.error('🚀 DBSearchPageFiltersComponent - ALL PROPS AND VARIABLES:'); + console.error('📋 Main Props:'); + console.error(' - analysisMode:', analysisMode); + console.error(' - isLive:', isLive); + console.error(' - sourceId:', sourceId); + console.error(' - sourceType:', sourceType); + console.error(' - showDelta:', showDelta); + console.error(' - denoiseResults:', denoiseResults); + console.error(' - chartConfig:', chartConfig); + console.error('📊 FilterStateHook Props:'); + console.error(' - filters (filterState):', filterState); + console.error(' - clearAllFilters function:', typeof clearAllFilters); + console.error(' - clearFilter function:', typeof clearFilter); + console.error(' - setFilterValue function:', typeof setFilterValue); + console.error('🔧 Hook Results:'); + const { toggleFilterPin, toggleFieldPin, isFilterPinned, isFieldPinned } = usePinnedFilters(sourceId ?? null); + + console.error(' - toggleFilterPin function:', typeof toggleFilterPin); + console.error(' - toggleFieldPin function:', typeof toggleFieldPin); + console.error(' - isFilterPinned function:', typeof isFilterPinned); + console.error(' - isFieldPinned function:', typeof isFieldPinned); + const { width, startResize } = useResizable(16, 'left'); + console.error(' - width:', width); + console.error(' - startResize function:', typeof startResize); const { data: countData } = useExplainQuery(chartConfig); @@ -444,7 +469,16 @@ const DBSearchPageFiltersComponent = ({ connectionId: chartConfig.connection, }); + console.error(' - countData:', countData); + console.error(' - allFields data:', data); + console.error(' - allFields isLoading:', isLoading); + console.error('📊 State Variables:'); + const [showMoreFields, setShowMoreFields] = useState(false); + console.error(' - showMoreFields:', showMoreFields); + console.error(' - setShowMoreFields function:', typeof setShowMoreFields); + console.error('🚀 END OF COMPONENT PROPS LOGGING 🚀'); + console.error(''); // const keysToFetch = useMemo(() => { // // Get keys for the source type - will return specific keys if available, or default keys otherwise @@ -488,7 +522,10 @@ const DBSearchPageFiltersComponent = ({ console.error('📊 Processing data for keysToFetch:'); console.error(' - Raw data sample:', data.slice(0, 3)); console.error(' - showMoreFields:', showMoreFields); - console.error(' - filterState keys:', Object.keys(DBSearchPageFiltersComponent)); + console.error( + ' - filterState keys:', + Object.keys(filterState), + ); const strings = data .sort((a, b) => { From e2db365a549ad840c548fb4e5772888862875688 Mon Sep 17 00:00:00 2001 From: nishant-sharechat Date: Mon, 11 Aug 2025 23:51:45 +0530 Subject: [PATCH 12/48] update --- packages/app/src/components/DBSearchPageFilters.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 30e5234ae..6448e82fd 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -522,10 +522,7 @@ const DBSearchPageFiltersComponent = ({ console.error('📊 Processing data for keysToFetch:'); console.error(' - Raw data sample:', data.slice(0, 3)); console.error(' - showMoreFields:', showMoreFields); - console.error( - ' - filterState keys:', - Object.keys(filterState), - ); + console.error(' - filterState keys:', Object.keys(filterState)); const strings = data .sort((a, b) => { From f9f7b64e7e8db6a32fa940d11e067ee9cf2e0741 Mon Sep 17 00:00:00 2001 From: Arpit Chhabra Date: Wed, 20 Aug 2025 15:43:56 +0530 Subject: [PATCH 13/48] hyper dx changes --- packages/app/src/DBSearchPage.tsx | 181 ++++++++++++++++-- packages/app/src/TeamPage.tsx | 20 +- .../src/components/DBSearchPageFilters.tsx | 95 +-------- .../app/src/components/SelectControlled.tsx | 8 +- packages/app/src/components/SourceSelect.tsx | 5 +- packages/app/src/config.ts | 3 + packages/app/src/hooks/useAuthEmails.tsx | 49 +++++ packages/app/src/hooks/useMetadata.tsx | 2 +- packages/common-utils/src/metadata.ts | 2 +- 9 files changed, 246 insertions(+), 119 deletions(-) create mode 100644 packages/app/src/hooks/useAuthEmails.tsx diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 6562b597a..e58a0989a 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -99,14 +99,15 @@ import { useSource, useSources, } from '@/source'; -import { parseTimeQuery, useNewTimeQuery } from '@/timeQuery'; +import { parseTimeQuery, useNewTimeQuery, dateRangeToString } from '@/timeQuery'; import { QUERY_LOCAL_STORAGE, useLocalStorage, usePrevious } from '@/utils'; +import { useAuthEmails } from '@/hooks/useAuthEmails'; import { SQLPreview } from './components/ChartSQLPreview'; import PatternTable from './components/PatternTable'; import { useSqlSuggestions } from './hooks/useSqlSuggestions'; import api from './api'; -import { LOCAL_STORE_CONNECTIONS_KEY } from './connection'; +import { LOCAL_STORE_CONNECTIONS_KEY, useConnections } from './connection'; import { DBSearchPageAlertModal } from './DBSearchPageAlertModal'; import { SearchConfig } from './types'; @@ -118,6 +119,7 @@ const SearchConfigSchema = z.object({ where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), + connection: z.string().optional(), filters: z.array( z.union([ z.object({ @@ -393,8 +395,13 @@ function SaveSearchModal({ ); } -// TODO: This is a hack to set the default time range -const defaultTimeRange = parseTimeQuery('Past 15m', false) as [Date, Date]; +// Create a fixed 15-minute time range that won't auto-update +const createFixedTimeRange = (): [Date, Date] => { + const end = new Date(); + const start = new Date(end.getTime() - 15 * 60 * 1000); // 15 minutes ago + return [start, end]; +}; +const defaultTimeRange = createFixedTimeRange(); function useLiveUpdate({ isLive, @@ -500,6 +507,7 @@ const queryStateMap = { whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), filters: parseAsJson(), orderBy: parseAsString, + connection: parseAsString, }; function DBSearchPage() { @@ -510,6 +518,29 @@ function DBSearchPage() { const [searchedConfig, setSearchedConfig] = useQueryStates(queryStateMap); + // On initial load, if URL contains filters param, strip it from URL and state + const initialHadFiltersRef = useRef( + (() => { + try { + const params = new URLSearchParams(window.location.search); + return params.has('filters'); + } catch { + return false; + } + })(), + ); + + useEffect(() => { + if (initialHadFiltersRef.current) { + try { + const url = new URL(window.location.href); + url.searchParams.delete('filters'); + window.history.replaceState({}, '', url.toString()); + } catch {} + setSearchedConfig({ filters: [] }); + } + }, [setSearchedConfig]); + const { data: savedSearch } = useSavedSearch( { id: `${savedSearchId}` }, { @@ -541,7 +572,7 @@ function DBSearchPage() { ); const [_isLive, setIsLive] = useQueryState('isLive', parseAsBoolean); - const isLive = _isLive ?? true; + const isLive = _isLive ?? false; // Default to false instead of true useEffect(() => { if (analysisMode === 'delta' || analysisMode === 'pattern') { @@ -567,6 +598,36 @@ function DBSearchPage() { [sources, lastSelectedSourceId], ); + const { data: connections } = useConnections(); + const { data: inputSourceObjs } = useSources(); + + // Get initial connection from URL or derive from source + const getInitialConnection = useCallback(() => { + // First try URL connection + if (searchedConfig.connection) { + return searchedConfig.connection; + } + + // Then try to get connection from current source + if (searchedConfig.source && inputSourceObjs) { + const sourceObj = inputSourceObjs.find(s => s.id === searchedConfig.source); + if (sourceObj?.connection) { + return sourceObj.connection; + } + } + + // Then try connection from last selected source + if (lastSelectedSourceId && inputSourceObjs) { + const lastSource = inputSourceObjs.find(s => s.id === lastSelectedSourceId); + if (lastSource?.connection) { + return lastSource.connection; + } + } + + // Finally fallback to first available connection + return connections?.[0]?.id || undefined; + }, [searchedConfig.connection, searchedConfig.source, inputSourceObjs, lastSelectedSourceId, connections]); + const { control, watch, @@ -577,7 +638,7 @@ function DBSearchPage() { formState, setError, resetField, - } = useForm({ + } = useForm({ values: { select: searchedConfig.select || '', where: searchedConfig.where || '', @@ -585,6 +646,7 @@ function DBSearchPage() { source: searchedConfig.source || defaultSourceId, filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', + connection: getInitialConnection(), }, resetOptions: { keepDirtyValues: true, @@ -594,8 +656,6 @@ function DBSearchPage() { }); const inputSource = watch('source'); - // const { data: inputSourceObj } = useSource({ id: inputSource }); - const { data: inputSourceObjs } = useSources(); const inputSourceObj = inputSourceObjs?.find(s => s.id === inputSource); // When source changes, make sure select and orderby fields are set to default @@ -609,14 +669,28 @@ function DBSearchPage() { const [rowId, setRowId] = useQueryState('rowWhere'); + // Create initial fixed time display value + const initialTimeDisplay = useMemo(() => { + return dateRangeToString(defaultTimeRange, false); + }, []); + const [displayedTimeInputValue, setDisplayedTimeInputValue] = - useState('Live Tail'); + useState(initialTimeDisplay); + + + + // Set isLive to false when starting with a fixed time range + useEffect(() => { + if (displayedTimeInputValue === initialTimeDisplay && _isLive == null) { + setIsLive(false); + } + }, [displayedTimeInputValue, _isLive, setIsLive, initialTimeDisplay]); const { from, to, isReady, searchedTimeRange, onSearch, onTimeRangeSelect } = useNewTimeQuery({ - initialDisplayValue: 'Live Tail', + initialDisplayValue: initialTimeDisplay, initialTimeRange: defaultTimeRange, - showRelativeInterval: isLive ?? true, + showRelativeInterval: isLive ?? false, setDisplayedTimeInputValue, updateInput: !isLive, }); @@ -646,9 +720,11 @@ function DBSearchPage() { source: searchedConfig?.source ?? undefined, filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', + // Use URL connection if available, otherwise preserve current form connection + connection: searchedConfig?.connection ?? watch('connection') ?? getInitialConnection(), }); } - }, [searchedConfig, reset, prevSearched]); + }, [searchedConfig, reset, prevSearched, getInitialConnection, watch]); // Populate searched query with saved search if the query params have // been wiped (ex. clicking on the same saved search again) @@ -715,7 +791,7 @@ function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ select, where, whereLanguage, source, filters, orderBy, connection }) => { setSearchedConfig({ select, where, @@ -723,6 +799,7 @@ function DBSearchPage() { source, filters, orderBy, + connection, }); }, )(); @@ -774,6 +851,20 @@ function DBSearchPage() { ); // Clear all search filters searchFilters.clearAllFilters(); + // Trigger refresh for new source + setTimeout(() => debouncedSubmit(), 0); + } + } + + // If the user changes the connection dropdown, update URL state and trigger data refresh + if (name === 'connection' && type === 'change') { + if (data.connection) { + setSearchedConfig(prev => ({ + ...prev, + connection: data.connection, + })); + // Trigger refresh for new connection + setTimeout(() => debouncedSubmit(), 0); } } }); @@ -785,6 +876,8 @@ function DBSearchPage() { inputSourceObjs, searchFilters, setLastSelectedSourceId, + setSearchedConfig, + debouncedSubmit, ]); const onTableScroll = useCallback( @@ -810,8 +903,17 @@ function DBSearchPage() { 'create' | 'update' | undefined >(undefined); + // Ignore initial filters from URL for first render to avoid narrowing facets + const searchedConfigForInit = useMemo( + () => + initialHadFiltersRef.current + ? { ...searchedConfig, filters: [] } + : searchedConfig, + [searchedConfig], + ); + const { data: chartConfig, isLoading: isChartConfigLoading } = - useSearchedConfigToChartConfig(searchedConfig); + useSearchedConfigToChartConfig(searchedConfigForInit); // query error handling const { hasQueryError, queryError } = useMemo(() => { @@ -996,7 +1098,8 @@ function DBSearchPage() { // Only trigger if we haven't searched yet (no time range in URL) const searchParams = new URLSearchParams(window.location.search); if (!searchParams.has('from') && !searchParams.has('to')) { - onSearch('Live Tail'); + let newTimeRange = new Date(defaultTimeRange[0]); + onSearch(newTimeRange.toISOString()); } } }, [isReady, queryReady, isChartConfigLoading, onSearch]); @@ -1103,8 +1206,45 @@ function DBSearchPage() { setNewSourceModalOpened(true); }, []); + // Parse demo auth emails from environment variable + const { authArray, hasAccess } = useAuthEmails(); + + // Ensure connection is set when data becomes available + useEffect(() => { + const currentConnection = watch('connection'); + if (!currentConnection && connections && connections.length > 0) { + const preferredConnectionId = getInitialConnection(); + if (preferredConnectionId) { + setValue('connection', preferredConnectionId); + // Update URL state as well + setSearchedConfig(prev => ({ + ...prev, + connection: preferredConnectionId, + })); + } + } + }, [connections, setValue, watch, getInitialConnection, setSearchedConfig]); + + // Ensure a matching initial source for the selected connection + const selectedConnectionId = watch('connection'); + useEffect(() => { + if (!selectedConnectionId || !inputSourceObjs) return; + const allowedKinds = [SourceKind.Log, SourceKind.Trace]; + const firstMatching = inputSourceObjs.find( + s => s.connection === selectedConnectionId && allowedKinds.includes(s.kind), + ); + if (!firstMatching) return; + const currentSourceId = watch('source'); + const currentSource = inputSourceObjs.find(s => s.id === (currentSourceId as any)); + if (!currentSource || currentSource.connection !== selectedConnectionId) { + setValue('source', firstMatching.id, { shouldDirty: true, shouldTouch: true }); + } + }, [selectedConnectionId, inputSourceObjs, setValue, watch]); + + return ( + {!IS_LOCAL_MODE && isAlertModalOpen && ( {/* */} + {/* + + */} - + + - + {authArray[me?.email as keyof typeof authArray] && ( + + )} Sources { if (email) { @@ -549,7 +551,7 @@ function TeamMembersSection() { )} - {!member.isCurrentUser && hasAdminAccess && ( + {!member.isCurrentUser && hasAdminAccess && hasAccess(me?.email) && (