Skip to content

Add harvester for Debian #46 #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlexandreBeaurain wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Add harvester for Debian #46 #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pygmars==0.7.0
Pygments==2.10.0
pymaven-patch==0.3.0
pyparsing==2.4.7
PyYAML==5.4.1
PyYAML==6.0.1
rdflib==6.0.1
requests==2.26.0
saneyaml==0.5.2
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/com/philips/research/bombase/core/debian/DebianException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyleft (c) 2024, Alexandre Beaurain
* SPDX-License-Identifier: MIT
*/

package com.philips.research.bombase.core.debian;

import com.philips.research.bombase.core.BusinessException;

public class DebianException extends BusinessException {
public DebianException(String message) {
super(message);
}

public DebianException(String message, Throwable cause) {
super(message, cause);
}
}
253 changes: 253 additions & 0 deletions src/main/java/com/philips/research/bombase/core/debian/domain/DebianAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
/*
* Copyleft (c) 2024, Alexandre Beaurain
* SPDX-License-Identifier: MIT
*/

package com.philips.research.bombase.core.debian.domain;

import com.philips.research.bombase.core.meta.PackageMetadata;
import com.philips.research.bombase.core.meta.registry.Field;
import com.philips.research.bombase.core.meta.registry.Trust;
import pl.tlinkowski.annotation.basic.NullOr;
import retrofit2.Call;
import retrofit2.http.GET;
import retrofit2.http.Path;
import retrofit2.http.Query;

import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;

public interface DebianAPI {
@GET("{distro}/series")
Call<SeriesCollection> series(
@Path("distro") String distro
);

@GET("{distro}/+archive/primary?exact_match=true&ws.op=getPublishedBinaries")
Call<SourcePackageCollection> getSourcePackages(
@Path("distro") String distro,
@Query("distro_arch_series") String distro_arch_series,
@Query("binary_name") String binary_name
);

@GET("{distro}/{series}/+source/{source}")
Call<Source> getSource(@Path("distro") String distro, @Path("series") String series, @Path("source") String source);

@GET("{project}")
Call<ResponseJson> getProject(@Path("project") String project);

@SuppressWarnings("NotNullFieldNotInitialized")
class Series {
URI self_link;
@NullOr URI web_link;
@NullOr URI resource_type_link;
@NullOr String bug_reporting_guidelines;
@NullOr String bug_reported_acknowledgement;
@NullOr List<String> official_bug_tags;
@NullOr URI active_milestones_collection_link;
@NullOr URI all_milestones_collection_link;
@NullOr boolean active;
@NullOr String summary;
@NullOr URI drivers_collection_link;
String name;
@NullOr String displayname;
@NullOr String fullseriesname;
@NullOr String title;
@NullOr String description;
@NullOr String version;
@NullOr URI distribution_link;
@NullOr List<String> component_names;
@NullOr List<String> suite_names;
@NullOr String status;
@NullOr Date datereleased;
@NullOr URI parent_series_link;
@NullOr URI registrant_link;
@NullOr URI owner_link;
@NullOr Date date_created;
@NullOr URI driver_link;
@NullOr String changeslist;
@NullOr URI nominatedarchindep_link;
@NullOr boolean language_pack_full_export_requested;
@NullOr boolean backports_not_automatic;
@NullOr boolean proposed_not_automatic;
@NullOr boolean include_long_descriptions;
@NullOr List<String> index_compressors;
@NullOr boolean publish_by_hash;
@NullOr boolean advertise_by_hash;
@NullOr boolean publish_i18n_index;
@NullOr URI main_archive_link;
@NullOr boolean supported;
@NullOr URI architectures_collection_link;
@NullOr String http_etag;
}

@SuppressWarnings("NotNullFieldNotInitialized")
class SeriesCollection {
int start;
int total_size;
List<Series> entries;
}

@SuppressWarnings("NotNullFieldNotInitialized")
class SourcePackageEntry {
URI self_link;
@NullOr URI resource_type_link;
@NullOr String display_name;
@NullOr String component_name;
@NullOr String section_name;
String source_package_name;
String source_package_version;
@NullOr URI distro_arch_series_link;
@NullOr String phased_update_percentage;
@NullOr Date date_published;
@NullOr Date scheduled_deletion_date;
@NullOr String status;
@NullOr String pocket;
@NullOr URI creator_link;
@NullOr Date date_created;
@NullOr Date date_superseded;
@NullOr Date date_made_pending;
@NullOr Date date_removed;
@NullOr URI archive_link;
@NullOr URI copied_from_archive_link;
@NullOr URI removed_by_link;
@NullOr String removal_comment;
String binary_package_name;
String binary_package_version;
@NullOr URI build_link;
@NullOr boolean architecture_specific;
@NullOr String priority_name;
@NullOr String http_etag;
}

@SuppressWarnings("NotNullFieldNotInitialized")
class SourcePackageCollection {
int start;
int total_size;
List<SourcePackageEntry> entries;
}

@SuppressWarnings("NotNullFieldNotInitialized")
class Source {
URI self_link;
@NullOr URI web_link;
@NullOr URI resource_type_link;
@NullOr String bug_reporting_guidelines;
@NullOr String bug_reported_acknowledgement;
@NullOr List<String> official_bug_tags;
String name;
@NullOr String displayname;
@NullOr URI distribution_link;
@NullOr URI distroseries_link;
URI productseries_link;
@NullOr String latest_published_component_name;
@NullOr String http_etag;
}

@SuppressWarnings("NotNullFieldNotInitialized")
class ResponseJson implements PackageMetadata {
URI self_link;
@NullOr URI web_link;
@NullOr URI resource_type_link;
@NullOr boolean official_answers;
@NullOr boolean official_blueprints;
@NullOr boolean official_codehosting;
@NullOr boolean official_bugs;
@NullOr String information_type;
@NullOr boolean active;
@NullOr String bug_reporting_guidelines;
@NullOr String bug_reported_acknowledgement;
@NullOr List<String> official_bug_tags;
@NullOr URI recipes_collection_link;
@NullOr URI webhooks_collection_link;
@NullOr URI bug_supervisor_link;
@NullOr URI active_milestones_collection_link;
@NullOr URI all_milestones_collection_link;
@NullOr boolean qualifies_for_free_hosting;
@NullOr String reviewer_whiteboard;
@NullOr String is_permitted;
@NullOr String project_reviewed;
@NullOr String license_approved;
@NullOr String display_name;
@NullOr URI icon_link;
@NullOr URI logo_link;
String name;
@NullOr URI owner_link;
@NullOr URI project_group_link;
@NullOr String title;
@NullOr URI registrant_link;
@NullOr URI driver_link;
@NullOr String summary;
@NullOr String description;
@NullOr Date date_created;
@NullOr URI homepage_url;
@NullOr URI wiki_url;
@NullOr URI screenshots_url;
@NullOr URI download_url;
@NullOr String programming_language;
@NullOr String sourceforge_project;
@NullOr String freshmeat_project;
@NullOr URI brand_link;
@NullOr boolean private_bugs;
List<String> licenses;
@NullOr String license_info;
@NullOr URI bug_tracker_link;
@NullOr String date_next_suggest_packaging;
@NullOr URI series_collection_link;
@NullOr URI development_focus_link;
@NullOr URI releases_collection_link;
@NullOr URI translation_focus_link;
@NullOr URI commercial_subscription_link;
@NullOr boolean commercial_subscription_is_due;
@NullOr String remote_product;
@NullOr String security_contact;
@NullOr String vcs;
@NullOr String http_etag;

@Override
public Trust trust(Field field) {
return Trust.LIKELY;
}

@Override
public Optional<String> getTitle() {
return Optional.ofNullable(name);
}

@Override
public Optional<String> getDescription() {
return Optional.ofNullable(description);
}

@Override
public Optional<List<String>> getAuthors() {
return Optional.of(new ArrayList<String>());
}

@Override
public Optional<URI> getHomepage() {
return Optional.ofNullable(homepage_url);
}

@Override
public Optional<String> getDeclaredLicense() {
if (licenses == null || licenses.size() == 0) {
return null;
}
return Optional.ofNullable(licenses.get(0).toString());
}

@Override
public Optional<String> getSourceLocation() {
return Optional.ofNullable(download_url != null ? download_url.toString() : null);
}

@Override
public Optional<URI> getDownloadLocation() {
return Optional.ofNullable(download_url);
}
}
}
100 changes: 100 additions & 0 deletions src/main/java/com/philips/research/bombase/core/debian/domain/DebianClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Copyleft (c) 2024, Alexandre Beaurain
* SPDX-License-Identifier: MIT
*/

package com.philips.research.bombase.core.debian.domain;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
import com.github.packageurl.PackageURL;
import com.philips.research.bombase.core.meta.PackageMetadata;
import com.philips.research.bombase.core.debian.DebianException;
import org.springframework.stereotype.Component;
import retrofit2.Call;
import retrofit2.Retrofit;
import retrofit2.converter.jackson.JacksonConverterFactory;

import java.io.IOException;
import java.net.URI;
import java.util.Optional;
import java.util.Map;

@Component
public class DebianClient {
private static final ObjectMapper MAPPER = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.NON_PRIVATE)
.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE);

private final DebianAPI rest;

private static final String baseApiUrl = "https://api.launchpad.net/1.0/";

DebianClient() {
this(URI.create(baseApiUrl));
}

DebianClient(URI uri) {
final var retrofit = new Retrofit.Builder()
.baseUrl(uri.toASCIIString())
.addConverterFactory(JacksonConverterFactory.create(MAPPER))
.build();
rest = retrofit.create(DebianAPI.class);
}

Optional<PackageMetadata> getPackageMetadata(PackageURL purl) {
final Map<String, String> qualifiers = purl.getQualifiers();
final String distro = "ubuntu";
final String packageName = purl.getName();
final var allSeries = query(rest.series(distro)).get().entries;
if (allSeries.size() == 0) {
return Optional.empty();
}
String seriesName = "noble";
for ( var serie : allSeries ) {
if (serie.active && serie.datereleased != null) {
seriesName = serie.name;
break;
}
}
String arch = qualifiers != null ? qualifiers.getOrDefault("arch", "amd64") : "amd64";
if (arch.equals("all")) {
arch = "amd64";
}
final String distroArchSeries = baseApiUrl + distro + "/" + seriesName + "/" + arch;
final var sourcePackagesCollection = query(rest.getSourcePackages(distro, '"' + distroArchSeries + '"', '"' + packageName + '"'));
final var sourcePackagesEntries = sourcePackagesCollection.get().entries;
if (sourcePackagesEntries.size() == 0) {
return Optional.empty();
}
final String sourceName = sourcePackagesEntries.get(0).source_package_name;
final var source = query(rest.getSource(distro, seriesName, sourceName));
if (source.isEmpty() || source.get() == null || source.get().productseries_link == null) {
return Optional.empty();
}
final String projectName = source.get().productseries_link.toString().replace(baseApiUrl, "").replaceAll("/.*", "");
return query(rest.getProject(projectName));
}

private <T> Optional<T> query(Call<? extends T> query) {
try {
final var response = query.execute();
if (response.code() == 404) {
return Optional.empty();
}
if (!response.isSuccessful()) {
throw new DebianException("Debian server responded with status " + response.code());
}
return Optional.ofNullable(response.body());
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("JSON formatting error", e);
} catch (IOException e) {
throw new DebianException("Debian is not reachable");
}
}
}
Loading