From 8df3dbef62df23c6dd7372bd75c41b25f19111eb Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Tue, 16 Dec 2025 15:00:02 +0100 Subject: [PATCH 1/9] wip --- pybana/client.py | 50 ++++++++----- pybana/elastic/elastic_client.py | 6 +- pybana/elastic/fixes_for_v8.py | 24 ++++--- pybana/models.py | 43 ++++++++---- pybana/translators/elastic/__init__.py | 10 ++- pybana/translators/vega/vega.py | 5 +- pybana/translators/vega/visualization.py | 4 +- tests/test_pybana.py | 89 ++++++++++++++++++------ 8 files changed, 162 insertions(+), 69 deletions(-) diff --git a/pybana/client.py b/pybana/client.py index 1fb662b..4805a30 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -23,16 +23,6 @@ } -def _fix_es(using): - if isinstance(using, ElasticsearchExtClient): - return using - using = using or "default" - es = elasticsearch_dsl.connections.get_connection(using) - if not isinstance(es, Elasticsearch): - return es - es_ext = ElasticsearchExtClient(es) - elasticsearch_dsl.connections.add_connection(using, es_ext) - return es_ext class Kibana: @@ -47,29 +37,44 @@ class Kibana: "search": Search, } - def __init__(self, index=".kibana"): + def get_es(self, using): + using = using or self._default + if isinstance(using, ElasticsearchExtClient): + return using + es = elasticsearch_dsl.connections.get_connection(using) + if not isinstance(es, Elasticsearch): + return es + es_ext = ElasticsearchExtClient(es) + assert isinstance(using, str) + elasticsearch_dsl.connections.add_connection(using, es_ext) + return es_ext + + def __init__(self, *, using, index=".kibana"): """ Initialize a client to kibana. :param index string: Index used by kibana (default: .kibana). """ + self._default = self.get_es(using) self._index = index def _search(self, type, using): klass = self.klasses.get(type) search = klass.search if klass else elasticsearch_dsl.Search - return search(index=self._index, using=_fix_es(using)) + es = self.get_es(using) + return search(index=self._index, using=es) def _get(self, klass, id, using): - ret = klass.get(index=self._index, id=id, using=_fix_es(using)) + ret = klass.get(index=self._index, id=id, using=self.get_es(using)) + return ret + setattr(ret, "_using", self.get_es(using)) return ret def objects(self, type, using=None): - return self._search(type, using=using).filter("term", type=type) + return self._search(type, using=using) def config_id(self, using=None): - elastic = _fix_es(using or "default") - # return "config:6.7.1" + elastic = self.get_es(using) return "config:%s" % elastic.info()["version"]["number"] def config(self, using=None): @@ -78,11 +83,16 @@ def config(self, using=None): """ return self._get(Config, self.config_id(using), using=using) - def init_index(self): + def is_v8(self, using=None): + elastic = self.get_es(using) + version = elastic.info()["version"]["number"].split(".")[0] + return version >= "8" + + def init_index(self, using=None): """ Create the elasticsearch index as kibana would do. """ - elastic = _fix_es("default") + elastic = self.get_es(using) mappingsfn = os.path.join(os.path.dirname(__file__), "mappings.json") suffix = 1 while not elastic.indices.exists(self._index): @@ -101,10 +111,12 @@ def init_config(self, using=None): date formats etc """ try: + print(f"init_config(using={using})") self.config(using=using) except NotFoundError: + print("NotFoundError: creating default config") Config(config=DEFAULT_CONFIG, meta={"id": self.config_id()}).save( - index=self._index, refresh="wait_for", using=using + index=self._index, refresh="wait_for", using=self.get_es(using) ) def index_patterns(self, using=None): diff --git a/pybana/elastic/elastic_client.py b/pybana/elastic/elastic_client.py index b52ffcd..bf64b73 100644 --- a/pybana/elastic/elastic_client.py +++ b/pybana/elastic/elastic_client.py @@ -67,7 +67,7 @@ def create( self._indices.create(index=index, body=body, **kwargs) return 1 - def refresh(self, index, **kwargs): # normally: only params + def refresh(self, index=None, **kwargs): # normally: only params return self._indices.refresh(index=index, **kwargs) def delete(self, index: Optional[str], **kwargs): # normally: only params @@ -403,14 +403,16 @@ def count( return self.es.count(index=index, doc_type=doc_type, body=body, **kwargs) def index(self, index, doc_type, body, id=None, **kwargs): + print(f"indexing doc in index={index}, doc_type={doc_type}, id={id}") if self.version_major >= 7: doc_type = "_doc" if "version" in kwargs and "version_type" not in kwargs: kwargs["version_type"] = "external" try: - return self.es.index( + r = self.es.index( index=index, doc_type=doc_type, body=body, id=id, **kwargs ) + return r except ConflictError: # increase the version of 1 for update if "version" in kwargs and isinstance(kwargs["version"], int): diff --git a/pybana/elastic/fixes_for_v8.py b/pybana/elastic/fixes_for_v8.py index 68f87d6..2a2b8c1 100644 --- a/pybana/elastic/fixes_for_v8.py +++ b/pybana/elastic/fixes_for_v8.py @@ -142,19 +142,27 @@ def _is_calendar_interval(interval: str) -> bool: class V6ToV8: def fix_transport_instance(self, transport: Transport): - _perform_request = transport.perform_request - transport._perform_request = _perform_request # type: ignore + if hasattr(transport, "_perform_request_v8"): + print("Transport already fixed for v8") + return def new_perform_request(method, url, headers=None, params=None, body=None): try: - return transport._perform_request( # type: ignore - method=method, url=url, headers=headers, params=params, body=body - ) - except TransportError as e: - if len(e.args) > 2: - e.args = v6_to_v8.fix_transport_error_args(e.args) + try: + #print(f"new_perform_request: method={method}, url={url}, params:{params}, body:{body}") + return transport._perform_request_v8( # type: ignore + method=method, url=url, headers=headers, params=params, body=body + ) + except TransportError as e: + if len(e.args) > 2: + e.args = v6_to_v8.fix_transport_error_args(e.args) + raise + except Exception as e: + print(f"ERROR in new_perform_request: method={method}, url={url}, params:{params}, body:{body}: {e}, hosts: {transport.hosts}") raise + _perform_request_v8 = transport.perform_request + transport._perform_request_v8 = _perform_request_v8 # type: ignore transport.perform_request = new_perform_request def fix_dynamic_template(self, dynamic: Dict) -> bool: diff --git a/pybana/models.py b/pybana/models.py index f9a907e..dc09a7a 100644 --- a/pybana/models.py +++ b/pybana/models.py @@ -4,11 +4,24 @@ from elasticsearch_dsl import Document, Keyword +from pybana.elastic.elastic_client import ElasticsearchExtClient + __all__ = ("BaseDocument", "Config", "IndexPattern", "Visualization", "Dashboard") class BaseDocument(Document): type = Keyword() + #_using = None + + @property + def using(self): + return self._using if hasattr(self, '_using') else 'default' + + @using.setter + def using(self, value): + self.__setattr__("_using", value) + #self._using = value + # List of json attributes. json_attrs = [] @@ -47,40 +60,40 @@ class IndexPattern(BaseDocument): class Search(BaseDocument): _type = "search" - def index(self): + def index(self, using=None): """ Returns the index-pattern associated to the visualization. Go through the search if needed. """ search_source = self.search["kibanaSavedObjectMeta"]["searchSourceJSON"] key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index) + return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using or self.using) class Visualization(BaseDocument): _type = "visualization" json_attrs = ["visState", "uiStateJSON"] - def related_search(self): + def related_search(self, using=None): """ Returns the search associated to the visualization. An error is raised if the visualization is not associated to any search. """ return Search.get( - id=f"search:{self.visualization.savedSearchId}", index=self.meta.index + id=f"search:{self.visualization.savedSearchId}", index=self.meta.index, using=using or self.using ) - def index(self): + def index(self, using=None): """ Returns the index-pattern associated to the visualization. Go through the search if needed. """ if hasattr(self.visualization, "savedSearchId"): - return self.related_search().index() + return self.related_search(using).index(using) search_source = self.visualization.kibanaSavedObjectMeta.searchSourceJSON key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index) + return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using or self.using) def filters(self): """ @@ -106,16 +119,19 @@ def visualizations(self, missing="skip", using=None): :param str using: connection alias to use, defaults to ``'default'`` """ panels = [panel for panel in self.panelsJSON if panel.type == "visualization"] - return ( + visu =( Visualization.mget( docs=["visualization:" + panel["id"] for panel in panels], index=self.meta.index, missing=missing, - using=using, + using=self.using, ) if panels else [] ) + for r in visu: + setattr(r, "_using", using or self.using) + return visu def searches(self, missing="skip", using=None): """ @@ -125,13 +141,16 @@ def searches(self, missing="skip", using=None): :param str using: connection alias to use, defaults to ``'default'`` """ panels = [panel for panel in self.panelsJSON if panel.type == "search"] - return ( + for r in ( Search.mget( docs=["search:" + panel["id"] for panel in panels], index=self.meta.index, missing=missing, - using=using, + using=using or self.using, ) if panels else [] - ) + ): + setattr(r, "_using", using or self.using) + yield r + diff --git a/pybana/translators/elastic/__init__.py b/pybana/translators/elastic/__init__.py index 33bed9e..403f28c 100644 --- a/pybana/translators/elastic/__init__.py +++ b/pybana/translators/elastic/__init__.py @@ -13,6 +13,10 @@ class ElasticTranslator: + + def __init__(self, using): + self._using = using + def translate_vega(self, visualization, scope): def replace_magic_keywords(node): if isinstance(node, list): @@ -66,7 +70,7 @@ def translate_data_item(data): index = data["url"]["index"] body = replace_magic_keywords(data["url"]["body"]) body = add_time_zone(body) - search = elasticsearch_dsl.Search(index=index).update_from_dict(body) + search = elasticsearch_dsl.Search(index=index, using=self._using).update_from_dict(body) if data["url"].get("%timefield%"): ts = data["url"]["%timefield%"] search = search.filter( @@ -79,7 +83,7 @@ def translate_data_item(data): } ) else: - search = elasticsearch_dsl.Search()[:0] + search = elasticsearch_dsl.Search(using=self._using)[:0] return search spec = hjson.loads(visualization.visState["params"]["spec"]) @@ -91,7 +95,7 @@ def translate_data_item(data): ) def translate_legacy(self, visualization, scope): - index_pattern = visualization.index() + index_pattern = visualization.index(using=self._using) fields = { field["name"]: field for field in json.loads(index_pattern["index-pattern"]["fields"]) diff --git a/pybana/translators/vega/vega.py b/pybana/translators/vega/vega.py index 74481c6..04546cb 100644 --- a/pybana/translators/vega/vega.py +++ b/pybana/translators/vega/vega.py @@ -25,6 +25,9 @@ class VegaTranslator: + def __init__(self, using): + self._using = using + def conf(self, state): return { "$schema": "https://vega.github.io/schema/vega/v5.json", @@ -998,7 +1001,7 @@ def marks_bar(self, conf, state, response): return conf def translate_legacy(self, visualization, response, scope): - state = ContextVisualization(visualization=visualization, config=scope.config) + state = ContextVisualization(visualization=visualization, config=scope.config, using=self._using) ret = self.conf(state) ret = self.data(ret, state, response, scope) diff --git a/pybana/translators/vega/visualization.py b/pybana/translators/vega/visualization.py index b2761ec..17bc5fe 100644 --- a/pybana/translators/vega/visualization.py +++ b/pybana/translators/vega/visualization.py @@ -14,8 +14,8 @@ class ContextVisualization: :param pybana.Config config: Config of the kibana instance. """ - def __init__(self, visualization, config): - self._index_pattern = visualization.index() + def __init__(self, visualization, config, using): + self._index_pattern = visualization.index(using=using) self._state = visualization.visState.to_dict() self._ui_state = visualization.uiStateJSON.to_dict() self._config = config diff --git a/tests/test_pybana.py b/tests/test_pybana.py index e8c3a72..10ddaa8 100644 --- a/tests/test_pybana.py +++ b/tests/test_pybana.py @@ -29,9 +29,18 @@ PYBANA_INDEX = ".kibana_pybana_test" -ELASTIC1 = elasticsearch.Elasticsearch() -ELASTIC = ElasticsearchExtClient() -elasticsearch_dsl.connections.add_connection("default", ELASTIC1) +ELASTICSEARCH_V6 = elasticsearch.Elasticsearch() +ELASTICSEARCH_V8 = elasticsearch.Elasticsearch(["http://localhost:9200"]) +ELASTIC_V6 = ElasticsearchExtClient(ELASTICSEARCH_V6) +ELASTIC_V8=ElasticsearchExtClient(ELASTICSEARCH_V8) +ELASTICS = { + "default": ELASTIC_V6, + "v6": ELASTIC_V6, + "v8": ELASTIC_V8, +} +elasticsearch_dsl.connections.add_connection("default", ELASTIC_V6) +elasticsearch_dsl.connections.add_connection("v6", ELASTIC_V6) +elasticsearch_dsl.connections.add_connection("v8", ELASTIC_V8) def load_fixtures(elastic, kibana, index): @@ -58,6 +67,7 @@ def load_data(elastic, index): ts = datetime.datetime(2019, 1, 1) if elastic.indices.exists(index): elastic.indices.delete(index) + print(f"creating index {index} for {elastic}") elastic.indices.create( index, body={ @@ -94,12 +104,18 @@ def actions(): elasticsearch.helpers.bulk(elastic, actions(), refresh="wait_for") +def test_client_v6(): + client_test("v6") +def test_client_v8(): + client_test("v8") -def test_client(): - kibana = Kibana(PYBANA_INDEX) - ELASTIC.indices.delete(f"{PYBANA_INDEX}*") - ELASTIC.indices.create(f"{PYBANA_INDEX}_1") - load_fixtures(ELASTIC, kibana, PYBANA_INDEX) + +def client_test(version): + kibana = Kibana(index=PYBANA_INDEX, using=version) + elastic= ELASTICS[version] + elastic.indices.delete(f"{PYBANA_INDEX}*") + elastic.indices.create(f"{PYBANA_INDEX}_1") + load_fixtures(elastic, kibana, PYBANA_INDEX) kibana.init_config() kibana.init_config() assert kibana.config() @@ -115,8 +131,9 @@ def test_client(): visualization = kibana.visualization("6eab7cb0-fb18-11e9-84e4-078763638bf3") visualization.visState visualization.uiStateJSON - assert visualization.index().meta.id == index_pattern.meta.id + assert visualization.index(using=elastic).meta.id == index_pattern.meta.id dashboards = list(kibana.dashboards()) + print(f"dashboards: {dashboards[0].__dict__}") assert len(dashboards) == 1 dashboard = kibana.dashboard("f57a7160-fb18-11e9-84e4-078763638bf3") dashboard.panelsJSON @@ -127,15 +144,31 @@ def test_client(): assert search.meta.id == "search:2139a4e0-fe77-11e9-833a-0fef2d7dd143" assert len(list(kibana.searches())) == 1 search = kibana.search("2139a4e0-fe77-11e9-833a-0fef2d7dd143") - assert visualization.index().meta.id == index_pattern.meta.id + assert visualization.index(using=elastic).meta.id == index_pattern.meta.id + +def test_translators_v6(): + translators_test("v6") + +def test_translators_v8(): + translators_test("v8") +def translators_test(version): + #elasticsearch_dsl.connections.add_connection("default", ELASTIC_V8) + + kibana = Kibana(index=PYBANA_INDEX, using=elasticsearch_dsl.connections.get_connection(version)) + elastic= ELASTICS[version] + assert isinstance(elasticsearch_dsl.connections.get_connection(version), ElasticsearchExtClient) + assert isinstance(elasticsearch_dsl.connections.get_connection(), ElasticsearchExtClient) + print(f"load_fixtures({elastic}, {kibana}, {PYBANA_INDEX})") + load_fixtures(elastic, kibana, PYBANA_INDEX) + print(f"load_data({elastic}, pybana)") + load_data(elastic, "pybana") + assert isinstance(elastic, ElasticsearchExtClient) -def test_translators(): - kibana = Kibana(PYBANA_INDEX) - load_fixtures(ELASTIC, kibana, PYBANA_INDEX) - load_data(ELASTIC, "pybana") kibana.init_config() - translator = ElasticTranslator() + #assert False + + translator = ElasticTranslator(using=elastic) scope = Scope( datetime.datetime(2019, 1, 1, tzinfo=pytz.utc), datetime.datetime(2019, 1, 3, tzinfo=pytz.utc), @@ -143,6 +176,7 @@ def test_translators(): kibana.config(), ) for visualization in kibana.visualizations().scan(): + print(f"visualization: {visualization.__class__.__name__} {visualization.__dict__}") if visualization.visState["type"] in ( "histogram", "metric", @@ -151,7 +185,11 @@ def test_translators(): "vega", "table", ): + print(f"visu: {visualization}") + #elastic.indices.refresh() + #print("after refresh") search = translator.translate(visualization, scope) + print("after translate") visualization_id = visualization.meta.id.split(":")[-1] if visualization_id in ( "695c02f0-fb1a-11e9-84e4-078763638bf3", @@ -172,8 +210,9 @@ def test_translators(): "5da362a0-732e-11ea-9c16-797f1f2fa4aa", "96645fc0-d636-11ea-8206-6f7030d7dd42", ): + print(f"search : {search.__class__.__name__} {search.__dict__}") response = search.execute() - VegaTranslator().translate(visualization, response, scope) + VegaTranslator(using=elastic).translate(visualization, response, scope) if visualization_id in ("d6c8b900-eea7-11eb-8e30-87c8d06ba6ff",): response = search.execute() metric = VEGA_METRICS["top_hits"]() @@ -190,13 +229,19 @@ def test_translators(): ret = metric.contribute(agg, response.aggregations, response) assert ret == results[agg["id"]] +def test_vega_visualization_v6(): + vega_visualization_test("v6") +def test_vega_visualization_v8(): + vega_visualization_test("v8") + +def vega_visualization_test(version): + kibana = Kibana(index=PYBANA_INDEX, using=version) + elastic= ELASTICS[version] -def test_vega_visualization(): - kibana = Kibana(PYBANA_INDEX) - load_fixtures(ELASTIC, kibana, PYBANA_INDEX) - load_data(ELASTIC, "pybana") + load_fixtures(elastic, kibana, PYBANA_INDEX) + load_data(elastic, "pybana") kibana.init_config() - translator = ElasticTranslator() + translator = ElasticTranslator(using=elastic) scope = Scope( datetime.datetime(2019, 1, 1, tzinfo=pytz.utc), datetime.datetime(2019, 1, 3, tzinfo=pytz.utc), @@ -216,7 +261,7 @@ def test_vega_visualization(): assert search_data["aggs"]["category"]["date_histogram"]["interval"] == "1h" response = search.execute() - VegaTranslator().translate(visualization, response, scope) + VegaTranslator(using=elastic).translate(visualization, response, scope) def test_elastic_translator_helpers(): From 65a08c0b683c0fa0c6d9f2a8ba6e7568d9655a35 Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Tue, 16 Dec 2025 16:37:29 +0100 Subject: [PATCH 2/9] for tests only --- pybana/client.py | 17 +++++----- pybana/elastic/elastic_client.py | 2 +- pybana/models.py | 42 ++++++++---------------- pybana/translators/vega/visualization.py | 2 +- tests/test_pybana.py | 10 +++--- 5 files changed, 28 insertions(+), 45 deletions(-) diff --git a/pybana/client.py b/pybana/client.py index 4805a30..4262c30 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -45,8 +45,8 @@ def get_es(self, using): if not isinstance(es, Elasticsearch): return es es_ext = ElasticsearchExtClient(es) - assert isinstance(using, str) - elasticsearch_dsl.connections.add_connection(using, es_ext) + if isinstance(using, str): + elasticsearch_dsl.connections.add_connection(using, es_ext) return es_ext def __init__(self, *, using, index=".kibana"): @@ -58,6 +58,10 @@ def __init__(self, *, using, index=".kibana"): self._default = self.get_es(using) self._index = index + @property + def using(self): + return self._default + def _search(self, type, using): klass = self.klasses.get(type) search = klass.search if klass else elasticsearch_dsl.Search @@ -65,13 +69,10 @@ def _search(self, type, using): return search(index=self._index, using=es) def _get(self, klass, id, using): - ret = klass.get(index=self._index, id=id, using=self.get_es(using)) - return ret - setattr(ret, "_using", self.get_es(using)) - return ret + return klass.get(index=self._index, id=id, using=self.get_es(using)) def objects(self, type, using=None): - return self._search(type, using=using) + return self._search(type, using=using).filter("term", type=type) def config_id(self, using=None): elastic = self.get_es(using) @@ -179,4 +180,4 @@ def update_or_create_default_index_pattern(self, index_pattern, using=None): config = self.config(using) if not config.config.to_dict().get("defaultIndex"): config.config.defaultIndex = index_pattern.meta.id.split(":")[-1] - config.save(refresh="wait_for") + config.save(refresh="wait_for", using=using or self.using) diff --git a/pybana/elastic/elastic_client.py b/pybana/elastic/elastic_client.py index bf64b73..c6f17a4 100644 --- a/pybana/elastic/elastic_client.py +++ b/pybana/elastic/elastic_client.py @@ -67,7 +67,7 @@ def create( self._indices.create(index=index, body=body, **kwargs) return 1 - def refresh(self, index=None, **kwargs): # normally: only params + def refresh(self, index, **kwargs): # normally: only params return self._indices.refresh(index=index, **kwargs) def delete(self, index: Optional[str], **kwargs): # normally: only params diff --git a/pybana/models.py b/pybana/models.py index dc09a7a..e4a9cc3 100644 --- a/pybana/models.py +++ b/pybana/models.py @@ -13,16 +13,6 @@ class BaseDocument(Document): type = Keyword() #_using = None - @property - def using(self): - return self._using if hasattr(self, '_using') else 'default' - - @using.setter - def using(self, value): - self.__setattr__("_using", value) - #self._using = value - - # List of json attributes. json_attrs = [] @@ -60,31 +50,31 @@ class IndexPattern(BaseDocument): class Search(BaseDocument): _type = "search" - def index(self, using=None): + def index(self, using): """ Returns the index-pattern associated to the visualization. Go through the search if needed. """ search_source = self.search["kibanaSavedObjectMeta"]["searchSourceJSON"] key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using or self.using) + return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using) class Visualization(BaseDocument): _type = "visualization" json_attrs = ["visState", "uiStateJSON"] - def related_search(self, using=None): + def related_search(self, using): """ Returns the search associated to the visualization. An error is raised if the visualization is not associated to any search. """ return Search.get( - id=f"search:{self.visualization.savedSearchId}", index=self.meta.index, using=using or self.using + id=f"search:{self.visualization.savedSearchId}", index=self.meta.index, using=using ) - def index(self, using=None): + def index(self, using): """ Returns the index-pattern associated to the visualization. Go through the search if needed. @@ -93,7 +83,7 @@ def index(self, using=None): return self.related_search(using).index(using) search_source = self.visualization.kibanaSavedObjectMeta.searchSourceJSON key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using or self.using) + return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using) def filters(self): """ @@ -111,7 +101,7 @@ class Dashboard(BaseDocument): _type = "dashboard" json_attrs = ["panelsJSON", "optionsJSON"] - def visualizations(self, missing="skip", using=None): + def visualizations(self, *, using, missing="skip"): """ Does the join automatically by parsing panelsJSON. @@ -119,21 +109,18 @@ def visualizations(self, missing="skip", using=None): :param str using: connection alias to use, defaults to ``'default'`` """ panels = [panel for panel in self.panelsJSON if panel.type == "visualization"] - visu =( + return ( Visualization.mget( docs=["visualization:" + panel["id"] for panel in panels], index=self.meta.index, missing=missing, - using=self.using, + using=using, ) if panels else [] ) - for r in visu: - setattr(r, "_using", using or self.using) - return visu - def searches(self, missing="skip", using=None): + def searches(self, *, using, missing="skip"): """ Does the join automatically by parsing panelsJSON. @@ -141,16 +128,13 @@ def searches(self, missing="skip", using=None): :param str using: connection alias to use, defaults to ``'default'`` """ panels = [panel for panel in self.panelsJSON if panel.type == "search"] - for r in ( + return ( Search.mget( docs=["search:" + panel["id"] for panel in panels], index=self.meta.index, missing=missing, - using=using or self.using, + using=using, ) if panels else [] - ): - setattr(r, "_using", using or self.using) - yield r - + ) diff --git a/pybana/translators/vega/visualization.py b/pybana/translators/vega/visualization.py index 17bc5fe..e7c31c7 100644 --- a/pybana/translators/vega/visualization.py +++ b/pybana/translators/vega/visualization.py @@ -14,7 +14,7 @@ class ContextVisualization: :param pybana.Config config: Config of the kibana instance. """ - def __init__(self, visualization, config, using): + def __init__(self, visualization, config, using=None): self._index_pattern = visualization.index(using=using) self._state = visualization.visState.to_dict() self._ui_state = visualization.uiStateJSON.to_dict() diff --git a/tests/test_pybana.py b/tests/test_pybana.py index 10ddaa8..e0071e1 100644 --- a/tests/test_pybana.py +++ b/tests/test_pybana.py @@ -30,8 +30,8 @@ PYBANA_INDEX = ".kibana_pybana_test" ELASTICSEARCH_V6 = elasticsearch.Elasticsearch() -ELASTICSEARCH_V8 = elasticsearch.Elasticsearch(["http://localhost:9200"]) -ELASTIC_V6 = ElasticsearchExtClient(ELASTICSEARCH_V6) +ELASTICSEARCH_V8 = elasticsearch.Elasticsearch(["http://localhost:9201"]) +ELASTIC_V6 = ElasticsearchExtClient() ELASTIC_V8=ElasticsearchExtClient(ELASTICSEARCH_V8) ELASTICS = { "default": ELASTIC_V6, @@ -138,9 +138,9 @@ def client_test(version): dashboard = kibana.dashboard("f57a7160-fb18-11e9-84e4-078763638bf3") dashboard.panelsJSON dashboard.optionsJSON - assert len(dashboard.visualizations()) == 2 + assert len(dashboard.visualizations(using=elastic)) == 2 visualization = kibana.visualization("f4a09a00-fe77-11e9-8c18-250a1adff826") - search = visualization.related_search() + search = visualization.related_search(using=elastic) assert search.meta.id == "search:2139a4e0-fe77-11e9-833a-0fef2d7dd143" assert len(list(kibana.searches())) == 1 search = kibana.search("2139a4e0-fe77-11e9-833a-0fef2d7dd143") @@ -157,8 +157,6 @@ def translators_test(version): kibana = Kibana(index=PYBANA_INDEX, using=elasticsearch_dsl.connections.get_connection(version)) elastic= ELASTICS[version] - assert isinstance(elasticsearch_dsl.connections.get_connection(version), ElasticsearchExtClient) - assert isinstance(elasticsearch_dsl.connections.get_connection(), ElasticsearchExtClient) print(f"load_fixtures({elastic}, {kibana}, {PYBANA_INDEX})") load_fixtures(elastic, kibana, PYBANA_INDEX) print(f"load_data({elastic}, pybana)") From efccc7855134438c57993319ea96d2f121c169a0 Mon Sep 17 00:00:00 2001 From: WilliamBinquet Date: Tue, 16 Dec 2025 16:42:25 +0100 Subject: [PATCH 3/9] Hotfix init_config() --- pybana/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pybana/client.py b/pybana/client.py index 4262c30..8ddc7f7 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -116,7 +116,9 @@ def init_config(self, using=None): self.config(using=using) except NotFoundError: print("NotFoundError: creating default config") - Config(config=DEFAULT_CONFIG, meta={"id": self.config_id()}).save( + Config( + config=DEFAULT_CONFIG, meta={"id": self.config_id(using=using)} + ).save( index=self._index, refresh="wait_for", using=self.get_es(using) ) From f90efa50c092c7c60dbe246b64294abd5f9587b2 Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Wed, 17 Dec 2025 07:28:57 +0100 Subject: [PATCH 4/9] black --- pybana/client.py | 6 +--- pybana/elastic/fixes_for_v8.py | 12 ++++++-- pybana/models.py | 14 ++++++--- pybana/translators/elastic/__init__.py | 5 ++-- pybana/translators/vega/vega.py | 4 ++- tests/test_pybana.py | 40 ++++++++++++++++---------- 6 files changed, 51 insertions(+), 30 deletions(-) diff --git a/pybana/client.py b/pybana/client.py index 8ddc7f7..4f0661f 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -23,8 +23,6 @@ } - - class Kibana: """ Kibana client. @@ -118,9 +116,7 @@ def init_config(self, using=None): print("NotFoundError: creating default config") Config( config=DEFAULT_CONFIG, meta={"id": self.config_id(using=using)} - ).save( - index=self._index, refresh="wait_for", using=self.get_es(using) - ) + ).save(index=self._index, refresh="wait_for", using=self.get_es(using)) def index_patterns(self, using=None): """ diff --git a/pybana/elastic/fixes_for_v8.py b/pybana/elastic/fixes_for_v8.py index 2a2b8c1..ea94dc6 100644 --- a/pybana/elastic/fixes_for_v8.py +++ b/pybana/elastic/fixes_for_v8.py @@ -149,16 +149,22 @@ def fix_transport_instance(self, transport: Transport): def new_perform_request(method, url, headers=None, params=None, body=None): try: try: - #print(f"new_perform_request: method={method}, url={url}, params:{params}, body:{body}") + # print(f"new_perform_request: method={method}, url={url}, params:{params}, body:{body}") return transport._perform_request_v8( # type: ignore - method=method, url=url, headers=headers, params=params, body=body + method=method, + url=url, + headers=headers, + params=params, + body=body, ) except TransportError as e: if len(e.args) > 2: e.args = v6_to_v8.fix_transport_error_args(e.args) raise except Exception as e: - print(f"ERROR in new_perform_request: method={method}, url={url}, params:{params}, body:{body}: {e}, hosts: {transport.hosts}") + print( + f"ERROR in new_perform_request: method={method}, url={url}, params:{params}, body:{body}: {e}, hosts: {transport.hosts}" + ) raise _perform_request_v8 = transport.perform_request diff --git a/pybana/models.py b/pybana/models.py index e4a9cc3..f60670e 100644 --- a/pybana/models.py +++ b/pybana/models.py @@ -11,7 +11,7 @@ class BaseDocument(Document): type = Keyword() - #_using = None + # _using = None # List of json attributes. json_attrs = [] @@ -57,7 +57,9 @@ def index(self, using): """ search_source = self.search["kibanaSavedObjectMeta"]["searchSourceJSON"] key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using) + return IndexPattern.get( + id=f"index-pattern:{key}", index=self.meta.index, using=using + ) class Visualization(BaseDocument): @@ -71,7 +73,9 @@ def related_search(self, using): An error is raised if the visualization is not associated to any search. """ return Search.get( - id=f"search:{self.visualization.savedSearchId}", index=self.meta.index, using=using + id=f"search:{self.visualization.savedSearchId}", + index=self.meta.index, + using=using, ) def index(self, using): @@ -83,7 +87,9 @@ def index(self, using): return self.related_search(using).index(using) search_source = self.visualization.kibanaSavedObjectMeta.searchSourceJSON key = json.loads(search_source).get("index") - return IndexPattern.get(id=f"index-pattern:{key}", index=self.meta.index, using=using) + return IndexPattern.get( + id=f"index-pattern:{key}", index=self.meta.index, using=using + ) def filters(self): """ diff --git a/pybana/translators/elastic/__init__.py b/pybana/translators/elastic/__init__.py index 403f28c..c97e21f 100644 --- a/pybana/translators/elastic/__init__.py +++ b/pybana/translators/elastic/__init__.py @@ -13,7 +13,6 @@ class ElasticTranslator: - def __init__(self, using): self._using = using @@ -70,7 +69,9 @@ def translate_data_item(data): index = data["url"]["index"] body = replace_magic_keywords(data["url"]["body"]) body = add_time_zone(body) - search = elasticsearch_dsl.Search(index=index, using=self._using).update_from_dict(body) + search = elasticsearch_dsl.Search( + index=index, using=self._using + ).update_from_dict(body) if data["url"].get("%timefield%"): ts = data["url"]["%timefield%"] search = search.filter( diff --git a/pybana/translators/vega/vega.py b/pybana/translators/vega/vega.py index 04546cb..0b06ff2 100644 --- a/pybana/translators/vega/vega.py +++ b/pybana/translators/vega/vega.py @@ -1001,7 +1001,9 @@ def marks_bar(self, conf, state, response): return conf def translate_legacy(self, visualization, response, scope): - state = ContextVisualization(visualization=visualization, config=scope.config, using=self._using) + state = ContextVisualization( + visualization=visualization, config=scope.config, using=self._using + ) ret = self.conf(state) ret = self.data(ret, state, response, scope) diff --git a/tests/test_pybana.py b/tests/test_pybana.py index e0071e1..1aace50 100644 --- a/tests/test_pybana.py +++ b/tests/test_pybana.py @@ -32,12 +32,8 @@ ELASTICSEARCH_V6 = elasticsearch.Elasticsearch() ELASTICSEARCH_V8 = elasticsearch.Elasticsearch(["http://localhost:9201"]) ELASTIC_V6 = ElasticsearchExtClient() -ELASTIC_V8=ElasticsearchExtClient(ELASTICSEARCH_V8) -ELASTICS = { - "default": ELASTIC_V6, - "v6": ELASTIC_V6, - "v8": ELASTIC_V8, -} +ELASTIC_V8 = ElasticsearchExtClient(ELASTICSEARCH_V8) +ELASTICS = {"default": ELASTIC_V6, "v6": ELASTIC_V6, "v8": ELASTIC_V8} elasticsearch_dsl.connections.add_connection("default", ELASTIC_V6) elasticsearch_dsl.connections.add_connection("v6", ELASTIC_V6) elasticsearch_dsl.connections.add_connection("v8", ELASTIC_V8) @@ -104,15 +100,18 @@ def actions(): elasticsearch.helpers.bulk(elastic, actions(), refresh="wait_for") + def test_client_v6(): client_test("v6") + + def test_client_v8(): client_test("v8") def client_test(version): kibana = Kibana(index=PYBANA_INDEX, using=version) - elastic= ELASTICS[version] + elastic = ELASTICS[version] elastic.indices.delete(f"{PYBANA_INDEX}*") elastic.indices.create(f"{PYBANA_INDEX}_1") load_fixtures(elastic, kibana, PYBANA_INDEX) @@ -146,17 +145,22 @@ def client_test(version): search = kibana.search("2139a4e0-fe77-11e9-833a-0fef2d7dd143") assert visualization.index(using=elastic).meta.id == index_pattern.meta.id + def test_translators_v6(): translators_test("v6") + def test_translators_v8(): translators_test("v8") + def translators_test(version): - #elasticsearch_dsl.connections.add_connection("default", ELASTIC_V8) + # elasticsearch_dsl.connections.add_connection("default", ELASTIC_V8) - kibana = Kibana(index=PYBANA_INDEX, using=elasticsearch_dsl.connections.get_connection(version)) - elastic= ELASTICS[version] + kibana = Kibana( + index=PYBANA_INDEX, using=elasticsearch_dsl.connections.get_connection(version) + ) + elastic = ELASTICS[version] print(f"load_fixtures({elastic}, {kibana}, {PYBANA_INDEX})") load_fixtures(elastic, kibana, PYBANA_INDEX) print(f"load_data({elastic}, pybana)") @@ -164,7 +168,7 @@ def translators_test(version): assert isinstance(elastic, ElasticsearchExtClient) kibana.init_config() - #assert False + # assert False translator = ElasticTranslator(using=elastic) scope = Scope( @@ -174,7 +178,9 @@ def translators_test(version): kibana.config(), ) for visualization in kibana.visualizations().scan(): - print(f"visualization: {visualization.__class__.__name__} {visualization.__dict__}") + print( + f"visualization: {visualization.__class__.__name__} {visualization.__dict__}" + ) if visualization.visState["type"] in ( "histogram", "metric", @@ -184,8 +190,8 @@ def translators_test(version): "table", ): print(f"visu: {visualization}") - #elastic.indices.refresh() - #print("after refresh") + # elastic.indices.refresh() + # print("after refresh") search = translator.translate(visualization, scope) print("after translate") visualization_id = visualization.meta.id.split(":")[-1] @@ -227,14 +233,18 @@ def translators_test(version): ret = metric.contribute(agg, response.aggregations, response) assert ret == results[agg["id"]] + def test_vega_visualization_v6(): vega_visualization_test("v6") + + def test_vega_visualization_v8(): vega_visualization_test("v8") + def vega_visualization_test(version): kibana = Kibana(index=PYBANA_INDEX, using=version) - elastic= ELASTICS[version] + elastic = ELASTICS[version] load_fixtures(elastic, kibana, PYBANA_INDEX) load_data(elastic, "pybana") From 9ff94eafd0e8b1028cb26e0b702d76b386f05079 Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Wed, 17 Dec 2025 08:05:49 +0100 Subject: [PATCH 5/9] add comment --- pybana/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybana/client.py b/pybana/client.py index 4f0661f..f502723 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -25,7 +25,7 @@ class Kibana: """ - Kibana client. + Kibana client (with support of multi elasticsearch). """ klasses = { From d54dba3ba442cf22912ed070bcd7b8ac161569bd Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Wed, 17 Dec 2025 08:44:36 +0100 Subject: [PATCH 6/9] flake8 --- pybana/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pybana/models.py b/pybana/models.py index f60670e..ed240a8 100644 --- a/pybana/models.py +++ b/pybana/models.py @@ -4,8 +4,6 @@ from elasticsearch_dsl import Document, Keyword -from pybana.elastic.elastic_client import ElasticsearchExtClient - __all__ = ("BaseDocument", "Config", "IndexPattern", "Visualization", "Dashboard") From 5da1c29b464d484a3dcac21759affeeb4493c09f Mon Sep 17 00:00:00 2001 From: gilles bourguignon Date: Wed, 17 Dec 2025 09:06:48 +0100 Subject: [PATCH 7/9] remove some print and other --- pybana/client.py | 2 -- pybana/elastic/elastic_client.py | 4 +--- pybana/elastic/fixes_for_v8.py | 22 ++++++---------------- tests/test_pybana.py | 12 ------------ 4 files changed, 7 insertions(+), 33 deletions(-) diff --git a/pybana/client.py b/pybana/client.py index f502723..f76deb5 100644 --- a/pybana/client.py +++ b/pybana/client.py @@ -110,10 +110,8 @@ def init_config(self, using=None): date formats etc """ try: - print(f"init_config(using={using})") self.config(using=using) except NotFoundError: - print("NotFoundError: creating default config") Config( config=DEFAULT_CONFIG, meta={"id": self.config_id(using=using)} ).save(index=self._index, refresh="wait_for", using=self.get_es(using)) diff --git a/pybana/elastic/elastic_client.py b/pybana/elastic/elastic_client.py index c6f17a4..b52ffcd 100644 --- a/pybana/elastic/elastic_client.py +++ b/pybana/elastic/elastic_client.py @@ -403,16 +403,14 @@ def count( return self.es.count(index=index, doc_type=doc_type, body=body, **kwargs) def index(self, index, doc_type, body, id=None, **kwargs): - print(f"indexing doc in index={index}, doc_type={doc_type}, id={id}") if self.version_major >= 7: doc_type = "_doc" if "version" in kwargs and "version_type" not in kwargs: kwargs["version_type"] = "external" try: - r = self.es.index( + return self.es.index( index=index, doc_type=doc_type, body=body, id=id, **kwargs ) - return r except ConflictError: # increase the version of 1 for update if "version" in kwargs and isinstance(kwargs["version"], int): diff --git a/pybana/elastic/fixes_for_v8.py b/pybana/elastic/fixes_for_v8.py index ea94dc6..5248f15 100644 --- a/pybana/elastic/fixes_for_v8.py +++ b/pybana/elastic/fixes_for_v8.py @@ -148,23 +148,13 @@ def fix_transport_instance(self, transport: Transport): def new_perform_request(method, url, headers=None, params=None, body=None): try: - try: - # print(f"new_perform_request: method={method}, url={url}, params:{params}, body:{body}") - return transport._perform_request_v8( # type: ignore - method=method, - url=url, - headers=headers, - params=params, - body=body, - ) - except TransportError as e: - if len(e.args) > 2: - e.args = v6_to_v8.fix_transport_error_args(e.args) - raise - except Exception as e: - print( - f"ERROR in new_perform_request: method={method}, url={url}, params:{params}, body:{body}: {e}, hosts: {transport.hosts}" + # print(f"new_perform_request: method={method}, url={url}, params:{params}, body:{body}") + return transport._perform_request_v8( # type: ignore + method=method, url=url, headers=headers, params=params, body=body ) + except TransportError as e: + if len(e.args) > 2: + e.args = v6_to_v8.fix_transport_error_args(e.args) raise _perform_request_v8 = transport.perform_request diff --git a/tests/test_pybana.py b/tests/test_pybana.py index 1aace50..fbd0418 100644 --- a/tests/test_pybana.py +++ b/tests/test_pybana.py @@ -63,7 +63,6 @@ def load_data(elastic, index): ts = datetime.datetime(2019, 1, 1) if elastic.indices.exists(index): elastic.indices.delete(index) - print(f"creating index {index} for {elastic}") elastic.indices.create( index, body={ @@ -132,7 +131,6 @@ def client_test(version): visualization.uiStateJSON assert visualization.index(using=elastic).meta.id == index_pattern.meta.id dashboards = list(kibana.dashboards()) - print(f"dashboards: {dashboards[0].__dict__}") assert len(dashboards) == 1 dashboard = kibana.dashboard("f57a7160-fb18-11e9-84e4-078763638bf3") dashboard.panelsJSON @@ -161,9 +159,7 @@ def translators_test(version): index=PYBANA_INDEX, using=elasticsearch_dsl.connections.get_connection(version) ) elastic = ELASTICS[version] - print(f"load_fixtures({elastic}, {kibana}, {PYBANA_INDEX})") load_fixtures(elastic, kibana, PYBANA_INDEX) - print(f"load_data({elastic}, pybana)") load_data(elastic, "pybana") assert isinstance(elastic, ElasticsearchExtClient) @@ -178,9 +174,6 @@ def translators_test(version): kibana.config(), ) for visualization in kibana.visualizations().scan(): - print( - f"visualization: {visualization.__class__.__name__} {visualization.__dict__}" - ) if visualization.visState["type"] in ( "histogram", "metric", @@ -189,11 +182,7 @@ def translators_test(version): "vega", "table", ): - print(f"visu: {visualization}") - # elastic.indices.refresh() - # print("after refresh") search = translator.translate(visualization, scope) - print("after translate") visualization_id = visualization.meta.id.split(":")[-1] if visualization_id in ( "695c02f0-fb1a-11e9-84e4-078763638bf3", @@ -214,7 +203,6 @@ def translators_test(version): "5da362a0-732e-11ea-9c16-797f1f2fa4aa", "96645fc0-d636-11ea-8206-6f7030d7dd42", ): - print(f"search : {search.__class__.__name__} {search.__dict__}") response = search.execute() VegaTranslator(using=elastic).translate(visualization, response, scope) if visualization_id in ("d6c8b900-eea7-11eb-8e30-87c8d06ba6ff",): From 1d680a8e9424bc092f97554374cc1e14e39ba613 Mon Sep 17 00:00:00 2001 From: WilliamBinquet Date: Mon, 5 Jan 2026 13:53:31 +0100 Subject: [PATCH 8/9] Pollux #11781 Hotfix vega preview with missing fields --- pybana/translators/vega/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybana/translators/vega/metrics.py b/pybana/translators/vega/metrics.py index 0f7b2a4..da103ab 100644 --- a/pybana/translators/vega/metrics.py +++ b/pybana/translators/vega/metrics.py @@ -93,7 +93,7 @@ def flatten(value): values = [ value for hit in bucket[agg["id"]]["hits"]["hits"] - for value in flatten(hit["_source"][agg["params"]["field"]]) + for value in flatten(hit["_source"].get(agg["params"]["field"])) if value is not None ] aggregate = agg["params"]["aggregate"] From f2e15a72177e891f147a1d5f1e17dfe8accf8099 Mon Sep 17 00:00:00 2001 From: WilliamBinquet Date: Mon, 5 Jan 2026 14:20:10 +0100 Subject: [PATCH 9/9] Not using .get() --- pybana/translators/vega/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pybana/translators/vega/metrics.py b/pybana/translators/vega/metrics.py index da103ab..0b93cc8 100644 --- a/pybana/translators/vega/metrics.py +++ b/pybana/translators/vega/metrics.py @@ -93,7 +93,8 @@ def flatten(value): values = [ value for hit in bucket[agg["id"]]["hits"]["hits"] - for value in flatten(hit["_source"].get(agg["params"]["field"])) + if agg["params"]["field"] in hit["_source"] + for value in flatten(hit["_source"][agg["params"]["field"]]) if value is not None ] aggregate = agg["params"]["aggregate"]