diff --git a/.env_template b/.env_template index 5af30fd8..85c172c3 100644 --- a/.env_template +++ b/.env_template @@ -13,4 +13,4 @@ COMPOSE_FILE=docker/default.yml:docker/local.override.yml #COMPOSE_FILE=docker/default.yml:docker/local.override.yml:docker/elasticsearch.yml # If you want to run a specific version, populate this -# REACT_APP_INTELOWL_VERSION="2.0.1" +# REACT_APP_INTELOWL_VERSION="2.1.0" diff --git a/.github/workflows/pull_request_automation.yml b/.github/workflows/pull_request_automation.yml index 03b7c81c..1ff60d09 100644 --- a/.github/workflows/pull_request_automation.yml +++ b/.github/workflows/pull_request_automation.yml @@ -100,6 +100,6 @@ jobs: "BROKER_URL": "amqp://guest:guest@rabbitmq:5672", } python_versions: >- - ["3.10"] + ["3.13"] max_timeout: 15 - ubuntu_version: 22.04 + ubuntu_version: 24.04 diff --git a/api/urls.py b/api/urls.py index 4dfffb59..ec341bcd 100644 --- a/api/urls.py +++ b/api/urls.py @@ -1,6 +1,15 @@ # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear # See the file 'LICENSE' for copying permission. -from api.views import StatisticsViewSet, command_sequence_view, enrichment_view, feeds, feeds_advanced, feeds_pagination, general_honeypot_list +from api.views import ( + StatisticsViewSet, + command_sequence_view, + cowrie_session_view, + enrichment_view, + feeds, + feeds_advanced, + feeds_pagination, + general_honeypot_list, +) from django.urls import include, path from rest_framework import routers @@ -14,6 +23,7 @@ path("feeds/advanced/", feeds_advanced), path("feeds///.", feeds), path("enrichment", enrichment_view), + path("cowrie_session", cowrie_session_view), path("command_sequence", command_sequence_view), path("general_honeypot", general_honeypot_list), # router viewsets diff --git a/api/views/__init__.py b/api/views/__init__.py index 2f5a7307..d76a0ef3 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -1,4 +1,5 @@ from api.views.command_sequence import * +from api.views.cowrie_session import * from api.views.enrichment import * from api.views.feeds import * from api.views.general_honeypot import * diff --git a/api/views/cowrie_session.py b/api/views/cowrie_session.py new file mode 100644 index 00000000..9ddb0b4c --- /dev/null +++ b/api/views/cowrie_session.py @@ -0,0 +1,119 @@ +# This file is a part of GreedyBear https://github.com/honeynet/GreedyBear +# See the file 'LICENSE' for copying permission. +import itertools +import logging +import socket + +from api.views.utils import is_ip_address, is_sha256hash +from certego_saas.apps.auth.backend import CookieTokenAuthentication +from django.http import Http404, HttpResponseBadRequest +from greedybear.consts import FEEDS_LICENSE, GET +from greedybear.models import IOC, CommandSequence, CowrieSession, Statistics, viewType +from rest_framework import status +from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +logger = logging.getLogger(__name__) + + +@api_view([GET]) +@authentication_classes([CookieTokenAuthentication]) +@permission_classes([IsAuthenticated]) +def cowrie_session_view(request): + """ + Retrieve Cowrie honeypot session data including command sequences, credentials, and session details. + Queries can be performed using either an IP address to find all sessions from that source, + or a SHA-256 hash to find sessions containing a specific command sequence. + + Args: + request: The HTTP request object containing query parameters + query (str, required): The search term, can be either an IP address or the SHA-256 hash of a command sequence. + SHA-256 hashes should match command sequences generated using Python's "\\n".join(sequence) format. + include_similar (bool, optional): When "true", expands the result to include all sessions that executed + command sequences belonging to the same cluster(s) as command sequences found in the initial query result. + Requires CLUSTER_COWRIE_COMMAND_SEQUENCES enabled in configuration. Default: false + include_credentials (bool, optional): When "true", includes all credentials used across matching Cowrie sessions. + Default: false + include_session_data (bool, optional): When "true", includes detailed information about matching Cowrie sessions. + Default: false + + Returns: + Response (200): JSON object containing: + - query (str): The original query parameter + - commands (list[str]): Unique command sequences (newline-delimited strings) + - sources (list[str]): Unique source IP addresses + - credentials (list[str], optional): Unique credentials if include_credentials=true + - sessions (list[dict], optional): Session details if include_session_data=true + - time (datetime): Session start time + - duration (float): Session duration in seconds + - source (str): Source IP address + - interactions (int): Number of interactions in session + - credentials (list[str]): Credentials used in this session + - commands (str): Command sequence executed (newline-delimited) + Response (400): Bad Request - Missing or invalid query parameter + Response (404): Not Found - No matching sessions found + Response (500): Internal Server Error - Unexpected error occurred + + Example Queries: + /api/cowrie_session?query=1.2.3.4 + /api/cowrie_session?query=5120e94e366ec83a79ee80454e4d1c76c06499ab19032bcdc7f0b4523bdb37a6 + /api/cowrie_session?query=1.2.3.4&include_credentials=true&include_session_data=true&include_similar=true + """ + observable = request.query_params.get("query") + include_similar = request.query_params.get("include_similar", "false").lower() == "true" + include_credentials = request.query_params.get("include_credentials", "false").lower() == "true" + include_session_data = request.query_params.get("include_session_data", "false").lower() == "true" + + logger.info(f"Cowrie view requested by {request.user} for {observable}") + source_ip = str(request.META["REMOTE_ADDR"]) + request_source = Statistics(source=source_ip, view=viewType.COWRIE_SESSION_VIEW.value) + request_source.save() + + if not observable: + return HttpResponseBadRequest("Missing required 'query' parameter") + + if is_ip_address(observable): + sessions = CowrieSession.objects.filter(source__name=observable, duration__gt=0).prefetch_related("source", "commands") + if not sessions.exists(): + raise Http404(f"No information found for IP: {observable}") + + elif is_sha256hash(observable): + try: + commands = CommandSequence.objects.get(commands_hash=observable.lower()) + except CommandSequence.DoesNotExist as exc: + raise Http404(f"No command sequences found with hash: {observable}") from exc + sessions = CowrieSession.objects.filter(commands=commands, duration__gt=0).prefetch_related("source", "commands") + else: + return HttpResponseBadRequest("Query must be a valid IP address or SHA-256 hash") + + if include_similar: + commands = set(s.commands for s in sessions if s.commands) + clusters = set(cmd.cluster for cmd in commands if cmd.cluster is not None) + related_sessions = CowrieSession.objects.filter(commands__cluster__in=clusters).prefetch_related("source", "commands") + sessions = sessions.union(related_sessions) + + response_data = { + "license": FEEDS_LICENSE, + "query": observable, + } + + unique_commands = set(s.commands for s in sessions if s.commands) + response_data["commands"] = sorted("\n".join(cmd.commands) for cmd in unique_commands) + response_data["sources"] = sorted(set(s.source.name for s in sessions), key=socket.inet_aton) + if include_credentials: + response_data["credentials"] = sorted(set(itertools.chain(*[s.credentials for s in sessions]))) + if include_session_data: + response_data["sessions"] = [ + { + "time": s.start_time, + "duration": s.duration, + "source": s.source.name, + "interactions": s.interaction_count, + "credentials": s.credentials if s.credentials else [], + "commands": "\n".join(s.commands.commands) if s.commands else "", + } + for s in sessions + ] + + return Response(response_data, status=status.HTTP_200_OK) diff --git a/docker/.version b/docker/.version index 45a727fb..f32f3526 100644 --- a/docker/.version +++ b/docker/.version @@ -1 +1 @@ -REACT_APP_GREEDYBEAR_VERSION="2.0.1" \ No newline at end of file +REACT_APP_GREEDYBEAR_VERSION="2.1.0" \ No newline at end of file diff --git a/docker/Dockerfile_nginx b/docker/Dockerfile_nginx index 55d4b221..c1a53bdf 100644 --- a/docker/Dockerfile_nginx +++ b/docker/Dockerfile_nginx @@ -1,4 +1,4 @@ -FROM library/nginx:1.29.1-alpine +FROM library/nginx:1.29.3-alpine RUN mkdir -p /var/cache/nginx /var/cache/nginx/feeds RUN apk update && apk upgrade && apk add bash ENV NGINX_LOG_DIR=/var/log/nginx diff --git a/greedybear/cronjobs/base.py b/greedybear/cronjobs/base.py index 9d83fa6d..c1928169 100644 --- a/greedybear/cronjobs/base.py +++ b/greedybear/cronjobs/base.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from django.conf import settings -from elasticsearch_dsl import Q, Search +from elasticsearch8.dsl import Q, Search from greedybear.settings import EXTRACTION_INTERVAL, LEGACY_EXTRACTION diff --git a/greedybear/models.py b/greedybear/models.py index 393a27a9..18e6c4f9 100644 --- a/greedybear/models.py +++ b/greedybear/models.py @@ -10,6 +10,7 @@ class viewType(models.TextChoices): FEEDS_VIEW = "feeds" ENRICHMENT_VIEW = "enrichment" COMMAND_SEQUENCE_VIEW = "command sequence" + COWRIE_SESSION_VIEW = "cowrie session" class iocType(models.TextChoices): diff --git a/greedybear/settings.py b/greedybear/settings.py index dcaa931a..cb986fbd 100644 --- a/greedybear/settings.py +++ b/greedybear/settings.py @@ -6,7 +6,7 @@ from datetime import timedelta from django.core.management.utils import get_random_secret_key -from elasticsearch import Elasticsearch +from elasticsearch8 import Elasticsearch BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_STATIC_PATH = os.path.join(BASE_DIR, "static/") diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index 15b2b6c1..f87af4a1 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -1,23 +1,23 @@ celery==5.5.3 # if you change this, update the documentation -elasticsearch-dsl==8.18.0 +elasticsearch8==8.19.2 Django==5.2.8 djangorestframework==3.16.1 django-rest-email-auth==5.0.0 django-ses==4.4.0 -psycopg2-binary==2.9.10 +psycopg2-binary==2.9.11 certego-saas==0.7.11 -slack-sdk==3.36.0 +slack-sdk==3.37.0 uwsgitop==0.12 -uwsgi==2.0.30 +uwsgi==2.0.31 joblib==1.5.2 pandas==2.3.3 -scikit-learn==1.6.1 -numpy==2.2.4 -datasketch==1.6.5 +scikit-learn==1.7.2 +numpy==2.3.5 +datasketch==1.7.0 diff --git a/tests/__init__.py b/tests/__init__.py index a84a7b98..31329e0d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -112,10 +112,37 @@ def setUpTestData(cls): ) cls.cowrie_session.save() + cls.cmd_seq_2 = ["cd bar", "ls -la"] + cls.command_sequence_2 = CommandSequence.objects.create( + first_seen=cls.current_time, + last_seen=cls.current_time, + commands=cls.cmd_seq_2, + commands_hash=sha256("\n".join(cls.cmd_seq_2).encode()).hexdigest(), + cluster=11, + ) + cls.command_sequence_2.save() + + cls.cowrie_session_2 = CowrieSession.objects.create( + session_id=int("eeeeeeeeeeee", 16), + start_time=cls.current_time, + duration=2.234, + login_attempt=True, + credentials=["user | user"], + command_execution=True, + interaction_count=5, + source=cls.ioc_2, + commands=cls.command_sequence_2, + ) + cls.cowrie_session_2.save() + try: cls.superuser = User.objects.get(is_superuser=True) except User.DoesNotExist: cls.superuser = User.objects.create_superuser(username="test", email="test@greedybear.com", password="test") + try: + cls.regular_user = User.objects.get(is_superuser=False) + except User.DoesNotExist: + cls.regular_user = User.objects.create_user(username="regular", email="regular@greedybear.com", password="regular") @classmethod def tearDownClass(self): diff --git a/tests/test_views.py b/tests/test_views.py index 6cabad7d..e6d9c587 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -298,6 +298,195 @@ def test_nonexistent_hash(self): self.assertEqual(response.status_code, 404) +class CowrieSessionViewTestCase(CustomTestCase): + """Test cases for the cowrie_session_view.""" + + def setUp(self): + # setup client + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + + # # # # # Basic IP Query Test # # # # # + def test_ip_address_query(self): + """Test view with a valid IP address query.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + + def test_ip_address_query_with_similar(self): + """Test view with a valid IP address query including similar sequences.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + self.assertEqual(len(response.data["sources"]), 2) + + def test_ip_address_query_with_credentials(self): + """Test view with a valid IP address query including credentials.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + self.assertEqual(len(response.data["credentials"]), 1) + self.assertEqual(response.data["credentials"][0], "root | root") + + def test_ip_address_query_with_sessions(self): + """Test view with a valid IP address query including session data.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertIn("sessions", response.data) + self.assertEqual(len(response.data["sessions"]), 1) + self.assertIn("time", response.data["sessions"][0]) + self.assertEqual(response.data["sessions"][0]["duration"], 1.234) + self.assertEqual(response.data["sessions"][0]["source"], "140.246.171.141") + self.assertEqual(response.data["sessions"][0]["interactions"], 5) + self.assertEqual(response.data["sessions"][0]["credentials"][0], "root | root") + self.assertEqual(response.data["sessions"][0]["commands"], "cd foo\nls -la") + + def test_ip_address_query_with_all(self): + """Test view with a valid IP address query including everything.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true&include_credentials=true&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertIn("sessions", response.data) + + # # # # # Basic Hash Query Test # # # # # + def test_hash_query(self): + """Test view with a valid hash query.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + + def test_hash_query_with_all(self): + """Test view with a valid hash query including everything.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}&include_similar=true&include_credentials=true&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertIn("sessions", response.data) + self.assertEqual(len(response.data["sources"]), 2) + + # # # # # IP Address Validation Tests # # # # # + def test_nonexistent_ip_address(self): + """Test that view returns 404 for IP with no sequences.""" + response = self.client.get("/api/cowrie_session?query=10.0.0.1") + self.assertEqual(response.status_code, 404) + + def test_ipv6_address_query(self): + """Test view with a valid IPv6 address query.""" + response = self.client.get("/api/cowrie_session?query=2001:db8::1") + self.assertEqual(response.status_code, 404) + + def test_invalid_ip_format(self): + """Test that malformed IP addresses are rejected.""" + response = self.client.get("/api/cowrie_session?query=999.999.999.999") + self.assertEqual(response.status_code, 400) + + def test_ip_with_cidr_notation(self): + """Test that CIDR notation is rejected.""" + response = self.client.get("/api/cowrie_session?query=192.168.1.0/24") + self.assertEqual(response.status_code, 400) + + # # # # # Parameter Validation Tests # # # # # + def test_missing_query_parameter(self): + """Test that view returns BadRequest when query parameter is missing.""" + response = self.client.get("/api/cowrie_session") + self.assertEqual(response.status_code, 400) + + def test_invalid_query_parameter(self): + """Test that view returns BadRequest when query parameter is invalid.""" + response = self.client.get("/api/cowrie_session?query=invalid-input}") + self.assertEqual(response.status_code, 400) + + def test_include_credentials_invalid_value(self): + """Test that invalid boolean values default to false.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=maybe") + self.assertEqual(response.status_code, 200) + self.assertNotIn("credentials", response.data) + + def test_case_insensitive_boolean_parameters(self): + """Test that boolean parameters accept various case formats.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=TRUE") + self.assertEqual(response.status_code, 200) + self.assertIn("credentials", response.data) + + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=True") + self.assertEqual(response.status_code, 200) + self.assertIn("credentials", response.data) + + # # # # # Hash Validation Tests # # # # # + def test_nonexistent_hash(self): + """Test that view returns 404 for nonexistent hash.""" + response = self.client.get(f"/api/cowrie_session?query={'f' * 64}") + self.assertEqual(response.status_code, 404) + + def test_hash_wrong_length(self): + """Test that hashes with incorrect length are rejected.""" + response = self.client.get("/api/cowrie_session?query=" + "a" * 32) # 32 chars instead of 64 + self.assertEqual(response.status_code, 400) + + def test_hash_invalid_characters(self): + """Test that hashes with invalid characters are rejected.""" + invalid_hash = "g" * 64 # 'g' is not a valid hex character + response = self.client.get(f"/api/cowrie_session?query={invalid_hash}") + self.assertEqual(response.status_code, 400) + + def test_hash_case_insensitive(self): + """Test that hash queries are case-insensitive.""" + response_lower = self.client.get(f"/api/cowrie_session?query={self.hash.lower()}") + response_upper = self.client.get(f"/api/cowrie_session?query={self.hash.upper()}") + self.assertEqual(response_lower.status_code, response_upper.status_code) + + # # # # # Special Characters & Encoding Tests # # # # # + def test_query_with_url_encoding(self): + """Test that URL-encoded queries work correctly.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141%20") + # Should either work or return 400, not crash + self.assertIn(response.status_code, [200, 400, 404]) + + def test_query_with_special_characters(self): + """Test handling of queries with special characters.""" + response = self.client.get("/api/cowrie_session?query=") + self.assertEqual(response.status_code, 400) + + # # # # # Authentication & Authorization Tests # # # # # + def test_unauthenticated_request(self): + """Test that unauthenticated requests are rejected.""" + client = APIClient() # No authentication + response = client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 401) + + def test_regular_user_access(self): + """Test that regular (non-superuser) authenticated users can access.""" + client = APIClient() + client.force_authenticate(user=self.regular_user) + response = client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + + class ValidationHelpersTestCase(CustomTestCase): """Test cases for the validation helper functions."""