From b57d3259958a21ad0821d963a2b485165e88fc32 Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 23 Dec 2025 17:33:40 +0800 Subject: [PATCH 1/6] add udf api --- .../java/org/apache/gravitino/Catalog.java | 9 + .../FunctionAlreadyExistsException.java | 50 +++ .../exceptions/NoSuchFunctionException.java | 49 +++ .../NoSuchFunctionVersionException.java | 49 +++ .../apache/gravitino/function/Function.java | 82 ++++ .../gravitino/function/FunctionCatalog.java | 147 +++++++ .../gravitino/function/FunctionChange.java | 392 ++++++++++++++++++ .../gravitino/function/FunctionColumn.java | 107 +++++ .../function/FunctionDefinition.java | 40 ++ .../function/FunctionDefinitions.java | 97 +++++ .../gravitino/function/FunctionImpl.java | 208 ++++++++++ .../gravitino/function/FunctionParam.java | 55 +++ .../gravitino/function/FunctionParams.java | 140 +++++++ .../gravitino/function/FunctionResources.java | 119 ++++++ .../gravitino/function/FunctionType.java | 72 ++++ .../apache/gravitino/function/JavaImpl.java | 84 ++++ .../apache/gravitino/function/PythonImpl.java | 98 +++++ .../apache/gravitino/function/SQLImpl.java | 84 ++++ 18 files changed, 1882 insertions(+) create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/FunctionAlreadyExistsException.java create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java create mode 100644 api/src/main/java/org/apache/gravitino/function/Function.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionChange.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionColumn.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionDefinition.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionImpl.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionParam.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionParams.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionResources.java create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionType.java create mode 100644 api/src/main/java/org/apache/gravitino/function/JavaImpl.java create mode 100644 api/src/main/java/org/apache/gravitino/function/PythonImpl.java create mode 100644 api/src/main/java/org/apache/gravitino/function/SQLImpl.java diff --git a/api/src/main/java/org/apache/gravitino/Catalog.java b/api/src/main/java/org/apache/gravitino/Catalog.java index 680c4c6b1ab..ed63285e95c 100644 --- a/api/src/main/java/org/apache/gravitino/Catalog.java +++ b/api/src/main/java/org/apache/gravitino/Catalog.java @@ -24,6 +24,7 @@ import org.apache.gravitino.authorization.SupportsRoles; import org.apache.gravitino.credential.SupportsCredentials; import org.apache.gravitino.file.FilesetCatalog; +import org.apache.gravitino.function.FunctionCatalog; import org.apache.gravitino.messaging.TopicCatalog; import org.apache.gravitino.model.ModelCatalog; import org.apache.gravitino.policy.SupportsPolicies; @@ -237,6 +238,14 @@ default ModelCatalog asModelCatalog() throws UnsupportedOperationException { throw new UnsupportedOperationException("Catalog does not support model operations"); } + /** + * @return the {@link FunctionCatalog} if the catalog supports function operations. + * @throws UnsupportedOperationException if the catalog does not support function operations. + */ + default FunctionCatalog asFunctionCatalog() throws UnsupportedOperationException { + throw new UnsupportedOperationException("Catalog does not support function operations"); + } + /** * @return the {@link SupportsTags} if the catalog supports tag operations. * @throws UnsupportedOperationException if the catalog does not support tag operations. diff --git a/api/src/main/java/org/apache/gravitino/exceptions/FunctionAlreadyExistsException.java b/api/src/main/java/org/apache/gravitino/exceptions/FunctionAlreadyExistsException.java new file mode 100644 index 00000000000..9e2cf69d9aa --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/FunctionAlreadyExistsException.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a function with a specified name already exists. */ +public class FunctionAlreadyExistsException extends AlreadyExistsException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public FunctionAlreadyExistsException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public FunctionAlreadyExistsException( + Throwable cause, @FormatString String message, Object... args) { + super(cause, message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java new file mode 100644 index 00000000000..537af8b4df6 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a function with specified name is not existed. */ +public class NoSuchFunctionException extends NotFoundException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchFunctionException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchFunctionException(Throwable cause, String message, Object... args) { + super(cause, message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java new file mode 100644 index 00000000000..8d6f49bbce2 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a function with specified version is not existed. */ +public class NoSuchFunctionVersionException extends NotFoundException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchFunctionVersionException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchFunctionVersionException(Throwable cause, String message, Object... args) { + super(cause, message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/Function.java b/api/src/main/java/org/apache/gravitino/function/Function.java new file mode 100644 index 00000000000..c0914297383 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/Function.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import javax.annotation.Nullable; +import org.apache.gravitino.Auditable; +import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.rel.types.Type; + +/** Represents a user-defined function registered in Gravitino. */ +@Evolving +public interface Function extends Auditable { + + /** + * @return The function name. + */ + String name(); + + /** + * @return The function type. + */ + FunctionType functionType(); + + /** + * @return Whether the function is deterministic. + */ + boolean deterministic(); + + /** + * @return The optional comment of the function. + */ + @Nullable + default String comment() { + return null; + } + + /** + * The return type for scalar or aggregate functions. + * + * @return The return type, null if this is a table-valued function. + */ + @Nullable + default Type returnType() { + return null; + } + + /** + * The output columns for a table-valued function. + * + * @return The output columns of a table-valued function, or an empty array for scalar or + * aggregate functions. + */ + default FunctionColumn[] returnColumns() { + return new FunctionColumn[0]; + } + + /** + * @return The definitions of the function. + */ + FunctionDefinition[] definitions(); + + /** + * @return The version of the function, counted from 0 and incrementing on each alteration. + */ + int version(); +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java new file mode 100644 index 00000000000..24357fdbc4a --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.rel.types.Type; + +/** The FunctionCatalog interface defines the public API for managing functions in a schema. */ +@Evolving +public interface FunctionCatalog { + + /** + * List the functions in a namespace from the catalog. + * + * @param namespace A namespace. + * @return An array of function identifiers in the namespace. + * @throws NoSuchSchemaException If the schema does not exist. + */ + NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException; + + /** + * Get a function by {@link NameIdentifier} from the catalog. The identifier only contains the + * schema and function name. A function may include multiple definitions (overloads) in the + * result. This method returns the latest version of the function. + * + * @param ident A function identifier. + * @return The latest version of the function with the given name. + * @throws NoSuchFunctionException If the function does not exist. + */ + Function getFunction(NameIdentifier ident) throws NoSuchFunctionException; + + /** + * Get a function by {@link NameIdentifier} and version from the catalog. The identifier only + * contains the schema and function name. A function may include multiple definitions (overloads) + * in the result. + * + * @param ident A function identifier. + * @param version The function version, counted from 0. + * @return The function with the given name and version. + * @throws NoSuchFunctionException If the function does not exist. + * @throws NoSuchFunctionVersionException If the function version does not exist. + */ + Function getFunction(NameIdentifier ident, int version) + throws NoSuchFunctionException, NoSuchFunctionVersionException; + + /** + * Check if a function with the given name exists in the catalog. + * + * @param ident The function identifier. + * @return True if the function exists, false otherwise. + */ + default boolean functionExists(NameIdentifier ident) { + try { + getFunction(ident); + return true; + } catch (NoSuchFunctionException e) { + return false; + } + } + + /** + * Register a scalar or aggregate function with one or more definitions (overloads). + * + * @param ident The function identifier. + * @param comment The optional function comment. + * @param functionType The function type. + * @param deterministic Whether the function is deterministic. + * @param returnType The return type. + * @param definitions The function definitions. + * @return The registered function. + * @throws NoSuchSchemaException If the schema does not exist. + * @throws FunctionAlreadyExistsException If the function already exists. + */ + Function registerFunction( + NameIdentifier ident, + String comment, + FunctionType functionType, + boolean deterministic, + Type returnType, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException; + + /** + * Register a table-valued function with one or more definitions (overloads). + * + * @param ident The function identifier. + * @param comment The optional function comment. + * @param deterministic Whether the function is deterministic. + * @param returnColumns The return columns. + * @param definitions The function definitions. + * @return The registered function. + * @throws NoSuchSchemaException If the schema does not exist. + * @throws FunctionAlreadyExistsException If the function already exists. + */ + Function registerFunction( + NameIdentifier ident, + String comment, + boolean deterministic, + FunctionColumn[] returnColumns, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException; + + /** + * Applies {@link FunctionChange changes} to a function in the catalog. + * + *

Implementations may reject the changes. If any change is rejected, no changes should be + * applied to the function. + * + * @param ident the {@link NameIdentifier} instance of the function to alter. + * @param changes the several {@link FunctionChange} instances to apply to the function. + * @return the updated {@link Function} instance. + * @throws NoSuchFunctionException If the function does not exist. + * @throws IllegalArgumentException If the change is rejected by the implementation. + */ + Function alterFunction(NameIdentifier ident, FunctionChange... changes) + throws NoSuchFunctionException, IllegalArgumentException; + + /** + * Drop a function by name. + * + * @param ident The name identifier of the function. + * @return True if the function is deleted, false if the function does not exist. + */ + boolean dropFunction(NameIdentifier ident); +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionChange.java b/api/src/main/java/org/apache/gravitino/function/FunctionChange.java new file mode 100644 index 00000000000..7bb5679edbd --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionChange.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.Objects; +import org.apache.gravitino.annotation.Evolving; + +/** Represents a change that can be applied to a function. */ +@Evolving +public interface FunctionChange { + + /** + * Create a {@link FunctionChange} to update the comment of a function. + * + * @param newComment The new comment value. + * @return The change instance. + */ + static FunctionChange updateComment(String newComment) { + return new UpdateComment(newComment); + } + + /** + * Create a {@link FunctionChange} to add a new definition (overload) to a function. + * + * @param definition The new definition to add. + * @return The change instance. + */ + static FunctionChange addDefinition(FunctionDefinition definition) { + return new AddDefinition(definition); + } + + /** + * Create a {@link FunctionChange} to remove an existing definition (overload) from a function. + * + * @param parameters The parameters that identify the definition to remove. + * @return The change instance. + */ + static FunctionChange removeDefinition(FunctionParam[] parameters) { + return new RemoveDefinition(parameters); + } + + /** + * Create a {@link FunctionChange} to add an implementation to a specific definition. + * + * @param parameters The parameters that identify the definition to update. + * @param implementation The implementation to add. + * @return The change instance. + */ + static FunctionChange addImpl(FunctionParam[] parameters, FunctionImpl implementation) { + return new AddImpl(parameters, implementation); + } + + /** + * Create a {@link FunctionChange} to update an implementation for a specific definition and + * runtime. + * + * @param parameters The parameters that identify the definition to update. + * @param runtime The runtime that identifies the implementation to replace. + * @param implementation The new implementation. + * @return The change instance. + */ + static FunctionChange updateImpl( + FunctionParam[] parameters, FunctionImpl.RuntimeType runtime, FunctionImpl implementation) { + return new UpdateImpl(parameters, runtime, implementation); + } + + /** + * Create a {@link FunctionChange} to remove an implementation for a specific definition and + * runtime. + * + * @param parameters The parameters that identify the definition to update. + * @param runtime The runtime that identifies the implementation to remove. + * @return The change instance. + */ + static FunctionChange removeImpl(FunctionParam[] parameters, FunctionImpl.RuntimeType runtime) { + return new RemoveImpl(parameters, runtime); + } + + /** A {@link FunctionChange} to update the comment of a function. */ + final class UpdateComment implements FunctionChange { + private final String newComment; + + UpdateComment(String newComment) { + this.newComment = newComment; + } + + /** + * @return The new comment of the function. + */ + public String newComment() { + return newComment; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof UpdateComment)) { + return false; + } + UpdateComment that = (UpdateComment) obj; + return Objects.equals(newComment, that.newComment); + } + + @Override + public int hashCode() { + return Objects.hash(newComment); + } + + @Override + public String toString() { + return "UpdateComment{newComment='" + newComment + "'}"; + } + } + + /** A {@link FunctionChange} to add a new definition to a function. */ + final class AddDefinition implements FunctionChange { + private final FunctionDefinition definition; + + AddDefinition(FunctionDefinition definition) { + this.definition = Preconditions.checkNotNull(definition, "Definition cannot be null"); + } + + /** + * @return The definition to add. + */ + public FunctionDefinition definition() { + return definition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof AddDefinition)) { + return false; + } + AddDefinition that = (AddDefinition) obj; + return Objects.equals(definition, that.definition); + } + + @Override + public int hashCode() { + return Objects.hash(definition); + } + + @Override + public String toString() { + return "AddDefinition{definition=" + definition + '}'; + } + } + + /** A {@link FunctionChange} to remove an existing definition from a function. */ + final class RemoveDefinition implements FunctionChange { + private final FunctionParam[] parameters; + + RemoveDefinition(FunctionParam[] parameters) { + Preconditions.checkArgument(parameters != null, "Parameters cannot be null"); + this.parameters = Arrays.copyOf(parameters, parameters.length); + } + + /** + * @return The parameters that identify the definition to remove. + */ + public FunctionParam[] parameters() { + return Arrays.copyOf(parameters, parameters.length); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RemoveDefinition)) { + return false; + } + RemoveDefinition that = (RemoveDefinition) obj; + return Arrays.equals(parameters, that.parameters); + } + + @Override + public int hashCode() { + return Arrays.hashCode(parameters); + } + + @Override + public String toString() { + return "RemoveDefinition{parameters=" + Arrays.toString(parameters) + '}'; + } + } + + /** A {@link FunctionChange} to add an implementation to a definition. */ + final class AddImpl implements FunctionChange { + private final FunctionParam[] parameters; + private final FunctionImpl implementation; + + AddImpl(FunctionParam[] parameters, FunctionImpl implementation) { + Preconditions.checkArgument(parameters != null, "Parameters cannot be null"); + this.parameters = Arrays.copyOf(parameters, parameters.length); + this.implementation = + Preconditions.checkNotNull(implementation, "Implementation cannot be null"); + } + + /** + * @return The parameters that identify the definition to update. + */ + public FunctionParam[] parameters() { + return Arrays.copyOf(parameters, parameters.length); + } + + /** + * @return The implementation to add. + */ + public FunctionImpl implementation() { + return implementation; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof AddImpl)) { + return false; + } + AddImpl that = (AddImpl) obj; + return Arrays.equals(parameters, that.parameters) + && Objects.equals(implementation, that.implementation); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(parameters); + result = 31 * result + Objects.hashCode(implementation); + return result; + } + + @Override + public String toString() { + return "AddImpl{parameters=" + + Arrays.toString(parameters) + + ", implementation=" + + implementation + + '}'; + } + } + + /** + * A {@link FunctionChange} to replace an implementation (identified by runtime) for a specific + * definition. + */ + final class UpdateImpl implements FunctionChange { + private final FunctionParam[] parameters; + private final FunctionImpl.RuntimeType runtime; + private final FunctionImpl implementation; + + UpdateImpl( + FunctionParam[] parameters, FunctionImpl.RuntimeType runtime, FunctionImpl implementation) { + Preconditions.checkArgument(parameters != null, "Parameters cannot be null"); + this.parameters = Arrays.copyOf(parameters, parameters.length); + this.runtime = Preconditions.checkNotNull(runtime, "Runtime cannot be null"); + this.implementation = + Preconditions.checkNotNull(implementation, "Implementation cannot be null"); + Preconditions.checkArgument( + runtime == implementation.runtime(), + "Runtime of implementation must match the runtime being updated"); + } + + /** + * @return The parameters that identify the definition to update. + */ + public FunctionParam[] parameters() { + return Arrays.copyOf(parameters, parameters.length); + } + + /** + * @return The runtime that identifies the implementation to replace. + */ + public FunctionImpl.RuntimeType runtime() { + return runtime; + } + + /** + * @return The new implementation. + */ + public FunctionImpl implementation() { + return implementation; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof UpdateImpl)) { + return false; + } + UpdateImpl that = (UpdateImpl) obj; + return Arrays.equals(parameters, that.parameters) + && runtime == that.runtime + && Objects.equals(implementation, that.implementation); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(parameters); + result = 31 * result + Objects.hash(runtime, implementation); + return result; + } + + @Override + public String toString() { + return "UpdateImpl{parameters=" + + Arrays.toString(parameters) + + ", runtime=" + + runtime + + ", implementation=" + + implementation + + '}'; + } + } + + /** A {@link FunctionChange} to remove an implementation for a specific runtime. */ + final class RemoveImpl implements FunctionChange { + private final FunctionParam[] parameters; + private final FunctionImpl.RuntimeType runtime; + + RemoveImpl(FunctionParam[] parameters, FunctionImpl.RuntimeType runtime) { + Preconditions.checkArgument(parameters != null, "Parameters cannot be null"); + this.parameters = Arrays.copyOf(parameters, parameters.length); + this.runtime = Preconditions.checkNotNull(runtime, "Runtime cannot be null"); + } + + /** + * @return The parameters that identify the definition to update. + */ + public FunctionParam[] parameters() { + return Arrays.copyOf(parameters, parameters.length); + } + + /** + * @return The runtime that identifies the implementation to remove. + */ + public FunctionImpl.RuntimeType runtime() { + return runtime; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RemoveImpl)) { + return false; + } + RemoveImpl that = (RemoveImpl) obj; + return Arrays.equals(parameters, that.parameters) && runtime == that.runtime; + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(parameters); + result = 31 * result + Objects.hashCode(runtime); + return result; + } + + @Override + public String toString() { + return "RemoveImpl{parameters=" + Arrays.toString(parameters) + ", runtime=" + runtime + '}'; + } + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java b/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java new file mode 100644 index 00000000000..0bbee04f34f --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import java.util.Objects; +import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.rel.types.Type; + +/** Represents a return column of a table-valued function. */ +@Evolving +public class FunctionColumn { + private final String name; + private final Type dataType; + private final String comment; + + private FunctionColumn(String name, Type dataType, String comment) { + Preconditions.checkArgument( + !Strings.isNullOrEmpty(name), "Function column name cannot be null"); + this.name = name; + this.dataType = Preconditions.checkNotNull(dataType, "Function column type cannot be null"); + this.comment = comment; + } + + /** + * Create a {@link FunctionColumn} instance. + * + * @param name The column name. + * @param dataType The column type. + * @param comment The optional comment of the column. + * @return A {@link FunctionColumn} instance. + */ + public static FunctionColumn of(String name, Type dataType, String comment) { + return new FunctionColumn(name, dataType, comment); + } + + /** + * @return The column name. + */ + public String name() { + return name; + } + + /** + * @return The column type. + */ + public Type dataType() { + return dataType; + } + + /** + * @return The optional column comment, null if not provided. + */ + public String comment() { + return comment; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FunctionColumn)) { + return false; + } + FunctionColumn that = (FunctionColumn) obj; + return Objects.equals(name, that.name) + && Objects.equals(dataType, that.dataType) + && Objects.equals(comment, that.comment); + } + + @Override + public int hashCode() { + return Objects.hash(name, dataType, comment); + } + + @Override + public String toString() { + return "FunctionColumn{" + + "name='" + + name + + '\'' + + ", dataType=" + + dataType + + ", comment='" + + comment + + '\'' + + '}'; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionDefinition.java b/api/src/main/java/org/apache/gravitino/function/FunctionDefinition.java new file mode 100644 index 00000000000..c7ea9562d48 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionDefinition.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import org.apache.gravitino.annotation.Evolving; + +/** + * A function definition that pairs a specific parameter list with its implementations. A single + * function can include multiple definitions (overloads), each with distinct parameters and + * implementations. + */ +@Evolving +public interface FunctionDefinition { + + /** + * @return The parameters for this definition. Maybe an empty array for a no-arg definition. + */ + FunctionParam[] parameters(); + + /** + * @return The implementations associated with this definition. + */ + FunctionImpl[] impls(); +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java new file mode 100644 index 00000000000..08dcb0133ea --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import java.util.Arrays; + +/** Helper methods to create {@link FunctionDefinition} instances. */ +public final class FunctionDefinitions { + + private FunctionDefinitions() {} + + /** + * Create a {@link FunctionDefinition} instance. + * + * @param parameters The parameters for this definition, maybe empty but not null. + * @param impls The implementations for this definition, it must not be null or empty. + * @return A {@link FunctionDefinition} instance. + */ + public static FunctionDefinition of(FunctionParam[] parameters, FunctionImpl[] impls) { + return new FunctionDefinitionImpl(parameters, impls); + } + + private static final class FunctionDefinitionImpl implements FunctionDefinition { + private final FunctionParam[] parameters; + private final FunctionImpl[] impls; + + FunctionDefinitionImpl(FunctionParam[] parameters, FunctionImpl[] impls) { + this.parameters = parameters == null ? new FunctionParam[0] : copy(parameters); + Preconditions.checkArgument( + impls != null && impls.length > 0, "Impls cannot be null or empty"); + this.impls = copy(impls); + } + + @Override + public FunctionParam[] parameters() { + return copy(parameters); + } + + @Override + public FunctionImpl[] impls() { + return copy(impls); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FunctionDefinitionImpl)) { + return false; + } + FunctionDefinitionImpl that = (FunctionDefinitionImpl) obj; + return Arrays.equals(parameters, that.parameters) && Arrays.equals(impls, that.impls); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(parameters); + result = 31 * result + Arrays.hashCode(impls); + return result; + } + + @Override + public String toString() { + return "FunctionDefinition{parameters=" + + Arrays.toString(parameters) + + ", impls=" + + Arrays.toString(impls) + + '}'; + } + + private static FunctionParam[] copy(FunctionParam[] params) { + return Arrays.copyOf(params, params.length); + } + + private static FunctionImpl[] copy(FunctionImpl[] impls) { + return Arrays.copyOf(impls, impls.length); + } + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java new file mode 100644 index 00000000000..f96e68198ca --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.annotation.Evolving; + +/** + * Base class of function implementations. + * + *

A function implementation must declare its language and optional external resources. Concrete + * implementations are provided by {@link SQLImpl}, {@link JavaImpl}, and {@link PythonImpl}. + */ +@Evolving +public abstract class FunctionImpl { + /** Supported implementation languages. */ + public enum Language { + /** SQL implementation. */ + SQL, + /** Java implementation. */ + JAVA, + /** Python implementation. */ + PYTHON + } + + /** Supported execution runtimes for function implementations. */ + public enum RuntimeType { + /** Spark runtime. */ + SPARK, + /** Trino runtime. */ + TRINO; + + /** + * Parse a runtime value from string. + * + * @param value Runtime name. + * @return Parsed runtime. + * @throws IllegalArgumentException If the runtime is not supported. + */ + public static RuntimeType fromString(String value) { + Preconditions.checkArgument(StringUtils.isNotBlank(value), "Function runtime must be set"); + try { + return RuntimeType.valueOf(value.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unsupported function runtime: " + value, e); + } + } + } + + private final Language language; + private final RuntimeType runtime; + private final FunctionResources resources; + private final Map properties; + + /** + * Construct a {@link FunctionImpl}. + * + * @param language The language of the function implementation. + * @param runtime The runtime of the function implementation. + * @param resources The resources required by the function implementation. + * @param properties The properties of the function implementation. + */ + protected FunctionImpl( + Language language, + RuntimeType runtime, + FunctionResources resources, + Map properties) { + Preconditions.checkNotNull(language, "Function implementation language must be set"); + Preconditions.checkNotNull(runtime, "Function runtime must be set"); + this.language = language; + this.runtime = runtime; + this.resources = resources == null ? FunctionResources.empty() : resources; + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + } + + /** + * Create a SQL implementation. + * + * @param runtime Target runtime. + * @param sql SQL text body. + * @return A {@link SQLImpl} instance. + */ + public static SQLImpl ofSql(RuntimeType runtime, String sql) { + return ofSql(runtime, sql, null, null); + } + + /** + * Create a SQL implementation. + * + * @param runtime Target runtime. + * @param sql SQL text body. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link SQLImpl} instance. + */ + public static SQLImpl ofSql( + RuntimeType runtime, + String sql, + FunctionResources resources, + Map properties) { + return new SQLImpl(runtime, sql, resources, properties); + } + + /** + * Create a Java implementation. + * + * @param runtime Target runtime. + * @param className Fully qualified class name. + * @return A {@link JavaImpl} instance. + */ + public static JavaImpl ofJava(RuntimeType runtime, String className) { + return ofJava(runtime, className, null, null); + } + + /** + * Create a Java implementation. + * + * @param runtime Target runtime. + * @param className Fully qualified class name. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link JavaImpl} instance. + */ + public static JavaImpl ofJava( + RuntimeType runtime, + String className, + FunctionResources resources, + Map properties) { + return new JavaImpl(runtime, className, resources, properties); + } + + /** + * Create a Python implementation. + * + * @param runtime Target runtime. + * @param handler Python handler entrypoint. + * @return A {@link PythonImpl} instance. + */ + public static PythonImpl ofPython(RuntimeType runtime, String handler) { + return ofPython(runtime, handler, null, null, null); + } + + /** + * Create a Python implementation. + * + * @param runtime Target runtime. + * @param handler Python handler entrypoint. + * @param codeBlock Inline code block for the handler. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link PythonImpl} instance. + */ + public static PythonImpl ofPython( + RuntimeType runtime, + String handler, + String codeBlock, + FunctionResources resources, + Map properties) { + return new PythonImpl(runtime, handler, codeBlock, resources, properties); + } + + /** + * @return The implementation language. + */ + public Language language() { + return language; + } + + /** + * @return The target runtime. + */ + public RuntimeType runtime() { + return runtime; + } + + /** + * @return The external resources required by this implementation. + */ + public FunctionResources resources() { + return resources; + } + + /** + * @return The additional properties of this implementation. + */ + public Map properties() { + return properties; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionParam.java b/api/src/main/java/org/apache/gravitino/function/FunctionParam.java new file mode 100644 index 00000000000..53172ca698d --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionParam.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; + +import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.rel.expressions.Expression; +import org.apache.gravitino.rel.types.Type; + +/** Represents a function parameter. */ +@Evolving +public interface FunctionParam { + + /** + * @return The name of the parameter. + */ + String name(); + + /** + * @return The data type of the parameter. + */ + Type dataType(); + + /** + * @return The optional comment of the parameter, null if not provided. + */ + default String comment() { + return null; + } + + /** + * @return The default value of the parameter if provided, otherwise {@link + * org.apache.gravitino.rel.Column#DEFAULT_VALUE_NOT_SET}. + */ + default Expression defaultValue() { + return DEFAULT_VALUE_NOT_SET; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionParams.java b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java new file mode 100644 index 00000000000..1150789ffb8 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import java.util.Objects; +import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.expressions.Expression; +import org.apache.gravitino.rel.types.Type; + +/** Helper methods to create {@link FunctionParam} instances. */ +public class FunctionParams { + + private FunctionParams() {} + + /** + * Create a {@link FunctionParam} instance. + * + * @param name The parameter name. + * @param dataType The parameter type. + * @return A {@link FunctionParam} instance. + */ + public static FunctionParam of(String name, Type dataType) { + return of(name, dataType, null, Column.DEFAULT_VALUE_NOT_SET); + } + + /** + * Create a {@link FunctionParam} instance with an optional comment. + * + * @param name The parameter name. + * @param dataType The parameter type. + * @param comment The optional comment. + * @return A {@link FunctionParam} instance. + */ + public static FunctionParam of(String name, Type dataType, String comment) { + return of(name, dataType, comment, Column.DEFAULT_VALUE_NOT_SET); + } + + /** + * Create a {@link FunctionParam} instance with an optional comment and default value. + * + * @param name The parameter name. + * @param dataType The parameter type. + * @param comment The optional comment. + * @param defaultValue The optional default value expression. + * @return A {@link FunctionParam} instance. + */ + public static FunctionParam of( + String name, Type dataType, String comment, Expression defaultValue) { + return new FunctionParamImpl(name, dataType, comment, defaultValue); + } + + private static final class FunctionParamImpl implements FunctionParam { + private final String name; + private final Type dataType; + private final String comment; + private final Expression defaultValue; + + private FunctionParamImpl(String name, Type dataType, String comment, Expression defaultValue) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Parameter name cannot be null"); + this.name = name; + this.dataType = Preconditions.checkNotNull(dataType, "Parameter data type cannot be null"); + this.comment = comment; + this.defaultValue = defaultValue == null ? Column.DEFAULT_VALUE_NOT_SET : defaultValue; + } + + @Override + public String name() { + return name; + } + + @Override + public Type dataType() { + return dataType; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Expression defaultValue() { + return defaultValue; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FunctionParamImpl)) { + return false; + } + FunctionParamImpl that = (FunctionParamImpl) obj; + return Objects.equals(name, that.name) + && Objects.equals(dataType, that.dataType) + && Objects.equals(comment, that.comment) + && Objects.equals(defaultValue, that.defaultValue); + } + + @Override + public int hashCode() { + return Objects.hash(name, dataType, comment, defaultValue); + } + + @Override + public String toString() { + return "FunctionParam{" + + "name='" + + name + + '\'' + + ", dataType=" + + dataType + + ", comment='" + + comment + + '\'' + + ", defaultValue=" + + defaultValue + + '}'; + } + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionResources.java b/api/src/main/java/org/apache/gravitino/function/FunctionResources.java new file mode 100644 index 00000000000..aa354975904 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionResources.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import java.util.Arrays; +import java.util.Objects; +import org.apache.gravitino.annotation.Evolving; + +/** Represents external resources that are required by a function implementation. */ +@Evolving +public class FunctionResources { + private static final String[] EMPTY = new String[0]; + private static final FunctionResources EMPTY_RESOURCES = + new FunctionResources(EMPTY, EMPTY, EMPTY); + + private final String[] jars; + private final String[] files; + private final String[] archives; + + private FunctionResources(String[] jars, String[] files, String[] archives) { + this.jars = jars == null ? EMPTY : Arrays.copyOf(jars, jars.length); + this.files = files == null ? EMPTY : Arrays.copyOf(files, files.length); + this.archives = archives == null ? EMPTY : Arrays.copyOf(archives, archives.length); + } + + /** + * @return An empty {@link FunctionResources} instance. + */ + public static FunctionResources empty() { + return EMPTY_RESOURCES; + } + + /** + * Create a {@link FunctionResources} instance. + * + * @param jars The jar resources. + * @param files The file resources. + * @param archives The archive resources. + * @return A {@link FunctionResources} instance. + */ + public static FunctionResources of(String[] jars, String[] files, String[] archives) { + if ((jars == null || jars.length == 0) + && (files == null || files.length == 0) + && (archives == null || archives.length == 0)) { + return EMPTY_RESOURCES; + } + return new FunctionResources(jars, files, archives); + } + + /** + * @return The jar resources. + */ + public String[] jars() { + return Arrays.copyOf(jars, jars.length); + } + + /** + * @return The file resources. + */ + public String[] files() { + return Arrays.copyOf(files, files.length); + } + + /** + * @return The archive resources. + */ + public String[] archives() { + return Arrays.copyOf(archives, archives.length); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FunctionResources)) { + return false; + } + FunctionResources that = (FunctionResources) obj; + return Arrays.equals(jars, that.jars) + && Arrays.equals(files, that.files) + && Arrays.equals(archives, that.archives); + } + + @Override + public int hashCode() { + int result = Objects.hash(Arrays.hashCode(jars), Arrays.hashCode(files)); + result = 31 * result + Arrays.hashCode(archives); + return result; + } + + @Override + public String toString() { + return "FunctionResources{" + + "jars=" + + Arrays.toString(jars) + + ", files=" + + Arrays.toString(files) + + ", archives=" + + Arrays.toString(archives) + + '}'; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionType.java b/api/src/main/java/org/apache/gravitino/function/FunctionType.java new file mode 100644 index 00000000000..e519eada0c7 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionType.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.function; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import org.apache.gravitino.annotation.Evolving; + +/** Function type supported by Gravitino. */ +@Evolving +public enum FunctionType { + /** Scalar function. */ + SCALAR, + + /** Aggregate function. */ + AGGREGATE, + + /** Table-valued function. */ + TABLE; + + private static final Map NAME_TO_TYPE; + + static { + Map map = new HashMap<>(); + for (FunctionType value : values()) { + map.put(value.typeName(), value); + } + map.put("agg", AGGREGATE); + NAME_TO_TYPE = Collections.unmodifiableMap(map); + } + + /** + * Parse the function type from a string value. + * + * @param type the string to parse. + * @return the parsed {@link FunctionType}. + * @throws IllegalArgumentException if the value cannot be parsed. + */ + public static FunctionType fromString(String type) { + FunctionType parsed = NAME_TO_TYPE.get(type.toLowerCase(Locale.ROOT)); + if (parsed == null) { + throw new IllegalArgumentException("Unknown function type: " + type); + } + return parsed; + } + + /** + * @return the canonical string representation used by APIs. + */ + public String typeName() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/JavaImpl.java b/api/src/main/java/org/apache/gravitino/function/JavaImpl.java new file mode 100644 index 00000000000..862f75f2cd1 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/JavaImpl.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import java.util.Map; +import java.util.Objects; + +/** Java implementation with class name. */ +public class JavaImpl extends FunctionImpl { + private final String className; + + JavaImpl( + RuntimeType runtime, + String className, + FunctionResources resources, + Map properties) { + super(Language.JAVA, runtime, resources, properties); + this.className = Preconditions.checkNotNull(className, "Java class name cannot be null"); + } + + /** + * @return The fully qualified class name. + */ + public String className() { + return className; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof JavaImpl)) { + return false; + } + JavaImpl that = (JavaImpl) obj; + return Objects.equals(language(), that.language()) + && Objects.equals(runtime(), that.runtime()) + && Objects.equals(resources(), that.resources()) + && Objects.equals(properties(), that.properties()) + && Objects.equals(className, that.className); + } + + @Override + public int hashCode() { + return Objects.hash(language(), runtime(), resources(), properties(), className); + } + + @Override + public String toString() { + return "JavaImpl{" + + "language='" + + language() + + '\'' + + ", runtime='" + + runtime() + + '\'' + + ", className='" + + className + + '\'' + + ", resources=" + + resources() + + ", properties=" + + properties() + + '}'; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/PythonImpl.java b/api/src/main/java/org/apache/gravitino/function/PythonImpl.java new file mode 100644 index 00000000000..9c7fabae1e5 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/PythonImpl.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import java.util.Map; +import java.util.Objects; + +/** Python implementation with handler and optional inline code. */ +public class PythonImpl extends FunctionImpl { + private final String handler; + private final String codeBlock; + + PythonImpl( + RuntimeType runtime, + String handler, + String codeBlock, + FunctionResources resources, + Map properties) { + super(Language.PYTHON, runtime, resources, properties); + this.handler = Preconditions.checkNotNull(handler, "Python handler cannot be null"); + this.codeBlock = codeBlock; + } + + /** + * @return The handler entrypoint. + */ + public String handler() { + return handler; + } + + /** + * @return The optional inline code block. + */ + public String codeBlock() { + return codeBlock; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof PythonImpl)) { + return false; + } + PythonImpl that = (PythonImpl) obj; + return Objects.equals(language(), that.language()) + && Objects.equals(runtime(), that.runtime()) + && Objects.equals(resources(), that.resources()) + && Objects.equals(properties(), that.properties()) + && Objects.equals(handler, that.handler) + && Objects.equals(codeBlock, that.codeBlock); + } + + @Override + public int hashCode() { + return Objects.hash(language(), runtime(), resources(), properties(), handler, codeBlock); + } + + @Override + public String toString() { + return "PythonImpl{" + + "language='" + + language() + + '\'' + + ", runtime='" + + runtime() + + '\'' + + ", handler='" + + handler + + '\'' + + ", codeBlock='" + + codeBlock + + '\'' + + ", resources=" + + resources() + + ", properties=" + + properties() + + '}'; + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/SQLImpl.java b/api/src/main/java/org/apache/gravitino/function/SQLImpl.java new file mode 100644 index 00000000000..a2a55f1cb53 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/SQLImpl.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import com.google.common.base.Preconditions; +import java.util.Map; +import java.util.Objects; + +/** SQL implementation with runtime and SQL body. */ +public class SQLImpl extends FunctionImpl { + private final String sql; + + SQLImpl( + RuntimeType runtime, + String sql, + FunctionResources resources, + Map properties) { + super(Language.SQL, runtime, resources, properties); + this.sql = Preconditions.checkNotNull(sql, "SQL text cannot be null"); + } + + /** + * @return The SQL body. + */ + public String sql() { + return sql; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SQLImpl)) { + return false; + } + SQLImpl that = (SQLImpl) obj; + return Objects.equals(language(), that.language()) + && Objects.equals(runtime(), that.runtime()) + && Objects.equals(resources(), that.resources()) + && Objects.equals(properties(), that.properties()) + && Objects.equals(sql, that.sql); + } + + @Override + public int hashCode() { + return Objects.hash(language(), runtime(), resources(), properties(), sql); + } + + @Override + public String toString() { + return "SQLImpl{" + + "language='" + + language() + + '\'' + + ", runtime='" + + runtime() + + '\'' + + ", sql='" + + sql + + '\'' + + ", resources=" + + resources() + + ", properties=" + + properties() + + '}'; + } +} From 852c53c7b06d41292e3325944bebd38ce738b8cc Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 30 Dec 2025 21:13:20 +0800 Subject: [PATCH 2/6] address comments from copilot --- .../exceptions/NoSuchFunctionException.java | 2 +- .../NoSuchFunctionVersionException.java | 2 +- .../apache/gravitino/function/Function.java | 11 ++++++++-- .../gravitino/function/FunctionCatalog.java | 3 ++- .../gravitino/function/FunctionChange.java | 1 + .../gravitino/function/FunctionColumn.java | 6 ++++-- .../function/FunctionDefinitions.java | 8 ++++---- .../gravitino/function/FunctionImpl.java | 6 +----- .../gravitino/function/FunctionParam.java | 2 ++ .../gravitino/function/FunctionParams.java | 20 ++++++++++--------- .../gravitino/function/FunctionResources.java | 6 ++---- .../apache/gravitino/function/PythonImpl.java | 2 +- .../apache/gravitino/function/SQLImpl.java | 2 +- 13 files changed, 40 insertions(+), 31 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java index 537af8b4df6..97a2d649203 100644 --- a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java +++ b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionException.java @@ -21,7 +21,7 @@ import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; -/** Exception thrown when a function with specified name is not existed. */ +/** Exception thrown when a function with the specified name does not exist. */ public class NoSuchFunctionException extends NotFoundException { /** diff --git a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java index 8d6f49bbce2..7c0f3a22cce 100644 --- a/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java +++ b/api/src/main/java/org/apache/gravitino/exceptions/NoSuchFunctionVersionException.java @@ -21,7 +21,7 @@ import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; -/** Exception thrown when a function with specified version is not existed. */ +/** Exception thrown when a function with the specified version does not exist. */ public class NoSuchFunctionVersionException extends NotFoundException { /** diff --git a/api/src/main/java/org/apache/gravitino/function/Function.java b/api/src/main/java/org/apache/gravitino/function/Function.java index c0914297383..d2d7d5073cc 100644 --- a/api/src/main/java/org/apache/gravitino/function/Function.java +++ b/api/src/main/java/org/apache/gravitino/function/Function.java @@ -26,6 +26,8 @@ /** Represents a user-defined function registered in Gravitino. */ @Evolving public interface Function extends Auditable { + /** An empty array of {@link FunctionColumn}. */ + FunctionColumn[] EMPTY = new FunctionColumn[0]; /** * @return The function name. @@ -67,7 +69,7 @@ default Type returnType() { * aggregate functions. */ default FunctionColumn[] returnColumns() { - return new FunctionColumn[0]; + return EMPTY; } /** @@ -76,7 +78,12 @@ default FunctionColumn[] returnColumns() { FunctionDefinition[] definitions(); /** - * @return The version of the function, counted from 0 and incrementing on each alteration. + * Returns the internal revision version of the function. + * + *

This version is a 0-based counter, where {@code 0} represents the initial definition of the + * function, and the value is incremented by 1 on each later alteration. + * + * @return The 0-based revision version of the function. */ int version(); } diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java index 24357fdbc4a..6cd7151b4b2 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java @@ -57,7 +57,8 @@ public interface FunctionCatalog { * in the result. * * @param ident A function identifier. - * @param version The function version, counted from 0. + * @param version The zero-based function version index (0 for the first created version), as + * returned by {@link Function#version()}. * @return The function with the given name and version. * @throws NoSuchFunctionException If the function does not exist. * @throws NoSuchFunctionVersionException If the function version does not exist. diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionChange.java b/api/src/main/java/org/apache/gravitino/function/FunctionChange.java index 7bb5679edbd..aa0d8acf860 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionChange.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionChange.java @@ -99,6 +99,7 @@ final class UpdateComment implements FunctionChange { private final String newComment; UpdateComment(String newComment) { + Preconditions.checkArgument(newComment != null, "New comment cannot be null"); this.newComment = newComment; } diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java b/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java index 0bbee04f34f..c93d056a83e 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionColumn.java @@ -19,8 +19,9 @@ package org.apache.gravitino.function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import java.util.Objects; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.rel.types.Type; @@ -33,7 +34,7 @@ public class FunctionColumn { private FunctionColumn(String name, Type dataType, String comment) { Preconditions.checkArgument( - !Strings.isNullOrEmpty(name), "Function column name cannot be null"); + StringUtils.isNotBlank(name), "Function column name cannot be null or empty"); this.name = name; this.dataType = Preconditions.checkNotNull(dataType, "Function column type cannot be null"); this.comment = comment; @@ -68,6 +69,7 @@ public Type dataType() { /** * @return The optional column comment, null if not provided. */ + @Nullable public String comment() { return comment; } diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java index 08dcb0133ea..926481ce2a7 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java @@ -29,7 +29,7 @@ private FunctionDefinitions() {} /** * Create a {@link FunctionDefinition} instance. * - * @param parameters The parameters for this definition, maybe empty but not null. + * @param parameters The parameters for this definition, it may be null or empty. * @param impls The implementations for this definition, it must not be null or empty. * @return A {@link FunctionDefinition} instance. */ @@ -63,11 +63,11 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof FunctionDefinitionImpl)) { + if (!(obj instanceof FunctionDefinition)) { return false; } - FunctionDefinitionImpl that = (FunctionDefinitionImpl) obj; - return Arrays.equals(parameters, that.parameters) && Arrays.equals(impls, that.impls); + FunctionDefinition that = (FunctionDefinition) obj; + return Arrays.equals(parameters, that.parameters()) && Arrays.equals(impls, that.impls()); } @Override diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java index f96e68198ca..f0244d09c4c 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java @@ -58,11 +58,7 @@ public enum RuntimeType { */ public static RuntimeType fromString(String value) { Preconditions.checkArgument(StringUtils.isNotBlank(value), "Function runtime must be set"); - try { - return RuntimeType.valueOf(value.trim().toUpperCase()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unsupported function runtime: " + value, e); - } + return RuntimeType.valueOf(value.trim().toUpperCase()); } } diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionParam.java b/api/src/main/java/org/apache/gravitino/function/FunctionParam.java index 53172ca698d..e0484451bea 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionParam.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionParam.java @@ -20,6 +20,7 @@ import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; +import javax.annotation.Nullable; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.rel.expressions.Expression; import org.apache.gravitino.rel.types.Type; @@ -41,6 +42,7 @@ public interface FunctionParam { /** * @return The optional comment of the parameter, null if not provided. */ + @Nullable default String comment() { return null; } diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionParams.java b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java index 1150789ffb8..0b2080737bc 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionParams.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java @@ -19,8 +19,8 @@ package org.apache.gravitino.function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.expressions.Expression; import org.apache.gravitino.rel.types.Type; @@ -74,9 +74,11 @@ private static final class FunctionParamImpl implements FunctionParam { private final Expression defaultValue; private FunctionParamImpl(String name, Type dataType, String comment, Expression defaultValue) { - Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Parameter name cannot be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "Parameter name cannot be null or empty"); + Preconditions.checkArgument(dataType != null, "Parameter data type cannot be null"); this.name = name; - this.dataType = Preconditions.checkNotNull(dataType, "Parameter data type cannot be null"); + this.dataType = dataType; this.comment = comment; this.defaultValue = defaultValue == null ? Column.DEFAULT_VALUE_NOT_SET : defaultValue; } @@ -106,14 +108,14 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof FunctionParamImpl)) { + if (!(obj instanceof FunctionParam)) { return false; } - FunctionParamImpl that = (FunctionParamImpl) obj; - return Objects.equals(name, that.name) - && Objects.equals(dataType, that.dataType) - && Objects.equals(comment, that.comment) - && Objects.equals(defaultValue, that.defaultValue); + FunctionParam that = (FunctionParam) obj; + return Objects.equals(name, that.name()) + && Objects.equals(dataType, that.dataType()) + && Objects.equals(comment, that.comment()) + && Objects.equals(defaultValue, that.defaultValue()); } @Override diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionResources.java b/api/src/main/java/org/apache/gravitino/function/FunctionResources.java index aa354975904..4a943856c86 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionResources.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionResources.java @@ -26,8 +26,6 @@ @Evolving public class FunctionResources { private static final String[] EMPTY = new String[0]; - private static final FunctionResources EMPTY_RESOURCES = - new FunctionResources(EMPTY, EMPTY, EMPTY); private final String[] jars; private final String[] files; @@ -43,7 +41,7 @@ private FunctionResources(String[] jars, String[] files, String[] archives) { * @return An empty {@link FunctionResources} instance. */ public static FunctionResources empty() { - return EMPTY_RESOURCES; + return new FunctionResources(EMPTY, EMPTY, EMPTY); } /** @@ -58,7 +56,7 @@ public static FunctionResources of(String[] jars, String[] files, String[] archi if ((jars == null || jars.length == 0) && (files == null || files.length == 0) && (archives == null || archives.length == 0)) { - return EMPTY_RESOURCES; + return new FunctionResources(EMPTY, EMPTY, EMPTY); } return new FunctionResources(jars, files, archives); } diff --git a/api/src/main/java/org/apache/gravitino/function/PythonImpl.java b/api/src/main/java/org/apache/gravitino/function/PythonImpl.java index 9c7fabae1e5..d4adcd87ed2 100644 --- a/api/src/main/java/org/apache/gravitino/function/PythonImpl.java +++ b/api/src/main/java/org/apache/gravitino/function/PythonImpl.java @@ -46,7 +46,7 @@ public String handler() { } /** - * @return The optional inline code block. + * @return The Python UDF code block. */ public String codeBlock() { return codeBlock; diff --git a/api/src/main/java/org/apache/gravitino/function/SQLImpl.java b/api/src/main/java/org/apache/gravitino/function/SQLImpl.java index a2a55f1cb53..2804f213f33 100644 --- a/api/src/main/java/org/apache/gravitino/function/SQLImpl.java +++ b/api/src/main/java/org/apache/gravitino/function/SQLImpl.java @@ -36,7 +36,7 @@ public class SQLImpl extends FunctionImpl { } /** - * @return The SQL body. + * @return The SQL that defines the function. */ public String sql() { return sql; From f5fdf136a8ff7e91062928173acfbf18c7331f26 Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 30 Dec 2025 21:17:56 +0800 Subject: [PATCH 3/6] fix --- .../gravitino/function/FunctionCatalog.java | 9 ++ .../function/FunctionDefinitions.java | 10 ++ .../gravitino/function/FunctionImpl.java | 86 ------------ .../gravitino/function/FunctionImpls.java | 122 ++++++++++++++++++ .../gravitino/function/FunctionParams.java | 11 ++ 5 files changed, 152 insertions(+), 86 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/function/FunctionImpls.java diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java index 6cd7151b4b2..73d9ce06ff0 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionCatalog.java @@ -40,6 +40,15 @@ public interface FunctionCatalog { */ NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException; + /** + * List the functions with details in a namespace from the catalog. + * + * @param namespace A namespace. + * @return An array of functions in the namespace. + * @throws NoSuchSchemaException If the schema does not exist. + */ + Function[] listFunctionInfos(Namespace namespace) throws NoSuchSchemaException; + /** * Get a function by {@link NameIdentifier} from the catalog. The identifier only contains the * schema and function name. A function may include multiple definitions (overloads) in the diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java index 926481ce2a7..4717f71173d 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionDefinitions.java @@ -26,6 +26,16 @@ public final class FunctionDefinitions { private FunctionDefinitions() {} + /** + * Create an array of {@link FunctionDefinition} instances. + * + * @param definitions The function definitions. + * @return An array of {@link FunctionDefinition} instances. + */ + public static FunctionDefinition[] of(FunctionDefinition... definitions) { + return Arrays.copyOf(definitions, definitions.length); + } + /** * Create a {@link FunctionDefinition} instance. * diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java index f0244d09c4c..2c4caa48b32 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionImpl.java @@ -88,92 +88,6 @@ protected FunctionImpl( this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); } - /** - * Create a SQL implementation. - * - * @param runtime Target runtime. - * @param sql SQL text body. - * @return A {@link SQLImpl} instance. - */ - public static SQLImpl ofSql(RuntimeType runtime, String sql) { - return ofSql(runtime, sql, null, null); - } - - /** - * Create a SQL implementation. - * - * @param runtime Target runtime. - * @param sql SQL text body. - * @param resources External resources required by the implementation. - * @param properties Additional implementation properties. - * @return A {@link SQLImpl} instance. - */ - public static SQLImpl ofSql( - RuntimeType runtime, - String sql, - FunctionResources resources, - Map properties) { - return new SQLImpl(runtime, sql, resources, properties); - } - - /** - * Create a Java implementation. - * - * @param runtime Target runtime. - * @param className Fully qualified class name. - * @return A {@link JavaImpl} instance. - */ - public static JavaImpl ofJava(RuntimeType runtime, String className) { - return ofJava(runtime, className, null, null); - } - - /** - * Create a Java implementation. - * - * @param runtime Target runtime. - * @param className Fully qualified class name. - * @param resources External resources required by the implementation. - * @param properties Additional implementation properties. - * @return A {@link JavaImpl} instance. - */ - public static JavaImpl ofJava( - RuntimeType runtime, - String className, - FunctionResources resources, - Map properties) { - return new JavaImpl(runtime, className, resources, properties); - } - - /** - * Create a Python implementation. - * - * @param runtime Target runtime. - * @param handler Python handler entrypoint. - * @return A {@link PythonImpl} instance. - */ - public static PythonImpl ofPython(RuntimeType runtime, String handler) { - return ofPython(runtime, handler, null, null, null); - } - - /** - * Create a Python implementation. - * - * @param runtime Target runtime. - * @param handler Python handler entrypoint. - * @param codeBlock Inline code block for the handler. - * @param resources External resources required by the implementation. - * @param properties Additional implementation properties. - * @return A {@link PythonImpl} instance. - */ - public static PythonImpl ofPython( - RuntimeType runtime, - String handler, - String codeBlock, - FunctionResources resources, - Map properties) { - return new PythonImpl(runtime, handler, codeBlock, resources, properties); - } - /** * @return The implementation language. */ diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionImpls.java b/api/src/main/java/org/apache/gravitino/function/FunctionImpls.java new file mode 100644 index 00000000000..72cf97adc98 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/function/FunctionImpls.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.function; + +import java.util.Arrays; +import java.util.Map; + +/** Helper methods to create {@link FunctionImpl} instances. */ +public class FunctionImpls { + + /** + * Copy an array of function implementations. + * + * @param impls The function implementations. + * @return A copy of the input array. + */ + public static FunctionImpl[] of(FunctionImpl... impls) { + return Arrays.copyOf(impls, impls.length); + } + + /** + * Create a SQL implementation. + * + * @param runtime Target runtime. + * @param sql SQL text body. + * @return A {@link SQLImpl} instance. + */ + public static SQLImpl ofSql(FunctionImpl.RuntimeType runtime, String sql) { + return ofSql(runtime, sql, null, null); + } + + /** + * Create a SQL implementation. + * + * @param runtime Target runtime. + * @param sql SQL text body. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link SQLImpl} instance. + */ + public static SQLImpl ofSql( + FunctionImpl.RuntimeType runtime, + String sql, + FunctionResources resources, + Map properties) { + return new SQLImpl(runtime, sql, resources, properties); + } + + /** + * Create a Java implementation. + * + * @param runtime Target runtime. + * @param className Fully qualified class name. + * @return A {@link JavaImpl} instance. + */ + public static JavaImpl ofJava(FunctionImpl.RuntimeType runtime, String className) { + return ofJava(runtime, className, null, null); + } + + /** + * Create a Java implementation. + * + * @param runtime Target runtime. + * @param className Fully qualified class name. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link JavaImpl} instance. + */ + public static JavaImpl ofJava( + FunctionImpl.RuntimeType runtime, + String className, + FunctionResources resources, + Map properties) { + return new JavaImpl(runtime, className, resources, properties); + } + + /** + * Create a Python implementation. + * + * @param runtime Target runtime. + * @param handler Python handler entrypoint. + * @return A {@link PythonImpl} instance. + */ + public static PythonImpl ofPython(FunctionImpl.RuntimeType runtime, String handler) { + return ofPython(runtime, handler, null, null, null); + } + + /** + * Create a Python implementation. + * + * @param runtime Target runtime. + * @param handler Python handler entrypoint. + * @param codeBlock Inline code block for the handler. + * @param resources External resources required by the implementation. + * @param properties Additional implementation properties. + * @return A {@link PythonImpl} instance. + */ + public static PythonImpl ofPython( + FunctionImpl.RuntimeType runtime, + String handler, + String codeBlock, + FunctionResources resources, + Map properties) { + return new PythonImpl(runtime, handler, codeBlock, resources, properties); + } +} diff --git a/api/src/main/java/org/apache/gravitino/function/FunctionParams.java b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java index 0b2080737bc..35263b19733 100644 --- a/api/src/main/java/org/apache/gravitino/function/FunctionParams.java +++ b/api/src/main/java/org/apache/gravitino/function/FunctionParams.java @@ -19,6 +19,7 @@ package org.apache.gravitino.function; import com.google.common.base.Preconditions; +import java.util.Arrays; import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.rel.Column; @@ -30,6 +31,16 @@ public class FunctionParams { private FunctionParams() {} + /** + * Create a copy of the given array of {@link FunctionParam} instances. + * + * @param params The array of parameters. + * @return A copy of the given array of {@link FunctionParam} instances. + */ + public static FunctionParam[] of(FunctionParam... params) { + return Arrays.copyOf(params, params.length); + } + /** * Create a {@link FunctionParam} instance. * From 85c9da24fe30008309ca8fa957dde9fd557d16ba Mon Sep 17 00:00:00 2001 From: mchades Date: Wed, 24 Dec 2025 23:07:56 +0800 Subject: [PATCH 4/6] feat(core): Add UDF management framework --- .../org/apache/gravitino/GravitinoEnv.java | 25 ++ .../gravitino/catalog/CapabilityHelpers.java | 6 +- .../gravitino/catalog/FunctionDispatcher.java | 29 ++ .../catalog/FunctionNormalizeDispatcher.java | 145 ++++++++++ .../catalog/FunctionOperationDispatcher.java | 254 ++++++++++++++++++ .../catalog/ManagedFunctionOperations.java | 137 ++++++++++ .../connector/capability/Capability.java | 7 +- 7 files changed, 600 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/catalog/FunctionDispatcher.java create mode 100644 core/src/main/java/org/apache/gravitino/catalog/FunctionNormalizeDispatcher.java create mode 100644 core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java create mode 100644 core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java index 34d55064209..8f89318389d 100644 --- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java +++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java @@ -34,6 +34,9 @@ import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.catalog.FilesetNormalizeDispatcher; import org.apache.gravitino.catalog.FilesetOperationDispatcher; +import org.apache.gravitino.catalog.FunctionDispatcher; +import org.apache.gravitino.catalog.FunctionNormalizeDispatcher; +import org.apache.gravitino.catalog.FunctionOperationDispatcher; import org.apache.gravitino.catalog.ModelDispatcher; import org.apache.gravitino.catalog.ModelNormalizeDispatcher; import org.apache.gravitino.catalog.ModelOperationDispatcher; @@ -126,6 +129,8 @@ public class GravitinoEnv { private ModelDispatcher modelDispatcher; + private FunctionDispatcher functionDispatcher; + private MetalakeDispatcher metalakeDispatcher; private CredentialOperationDispatcher credentialOperationDispatcher; @@ -253,6 +258,15 @@ public ModelDispatcher modelDispatcher() { return modelDispatcher; } + /** + * Get the FunctionDispatcher associated with the Gravitino environment. + * + * @return The FunctionDispatcher instance. + */ + public FunctionDispatcher functionDispatcher() { + return functionDispatcher; + } + /** * Get the PartitionDispatcher associated with the Gravitino environment. * @@ -578,6 +592,17 @@ private void initGravitinoServerComponents() { ModelNormalizeDispatcher modelNormalizeDispatcher = new ModelNormalizeDispatcher(modelHookDispatcher, catalogManager); this.modelDispatcher = new ModelEventDispatcher(eventBus, modelNormalizeDispatcher); + + // TODO: Add FunctionHookDispatcher and FunctionEventDispatcher when needed + // The operation chain should be: + // FunctionEventDispatcher -> FunctionNormalizeDispatcher -> FunctionHookDispatcher -> + // FunctionOperationDispatcher + FunctionOperationDispatcher functionOperationDispatcher = + new FunctionOperationDispatcher( + catalogManager, schemaOperationDispatcher, entityStore, idGenerator); + this.functionDispatcher = + new FunctionNormalizeDispatcher(functionOperationDispatcher, catalogManager); + this.statisticDispatcher = new StatisticEventDispatcher( eventBus, new StatisticManager(entityStore, idGenerator, config)); diff --git a/core/src/main/java/org/apache/gravitino/catalog/CapabilityHelpers.java b/core/src/main/java/org/apache/gravitino/catalog/CapabilityHelpers.java index fe4659c28e8..e405cfdd92f 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CapabilityHelpers.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CapabilityHelpers.java @@ -131,7 +131,8 @@ public static Namespace applyCaseSensitive( if (identScope == Capability.Scope.TABLE || identScope == Capability.Scope.FILESET || identScope == Capability.Scope.TOPIC - || identScope == Capability.Scope.MODEL) { + || identScope == Capability.Scope.MODEL + || identScope == Capability.Scope.FUNCTION) { String schema = namespace.level(namespace.length() - 1); schema = applyCaseSensitiveOnName(Capability.Scope.SCHEMA, schema, capabilities); return Namespace.of(metalake, catalog, schema); @@ -201,7 +202,8 @@ public static Namespace applyCapabilities( String catalog = namespace.level(1); if (identScope == Capability.Scope.TABLE || identScope == Capability.Scope.FILESET - || identScope == Capability.Scope.TOPIC) { + || identScope == Capability.Scope.TOPIC + || identScope == Capability.Scope.FUNCTION) { String schema = namespace.level(namespace.length() - 1); schema = applyCapabilitiesOnName(Capability.Scope.SCHEMA, schema, capabilities); return Namespace.of(metalake, catalog, schema); diff --git a/core/src/main/java/org/apache/gravitino/catalog/FunctionDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FunctionDispatcher.java new file mode 100644 index 00000000000..65c23c920c2 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/FunctionDispatcher.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import org.apache.gravitino.function.FunctionCatalog; + +/** + * {@code FunctionDispatcher} interface acts as a specialization of the {@link FunctionCatalog} + * interface. This interface is designed to potentially add custom behaviors or operations related + * to dispatching or handling function-related events or actions that are not covered by the + * standard {@code FunctionCatalog} operations. + */ +public interface FunctionDispatcher extends FunctionCatalog {} diff --git a/core/src/main/java/org/apache/gravitino/catalog/FunctionNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FunctionNormalizeDispatcher.java new file mode 100644 index 00000000000..4930a780aa8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/FunctionNormalizeDispatcher.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import static org.apache.gravitino.catalog.CapabilityHelpers.applyCapabilities; +import static org.apache.gravitino.catalog.CapabilityHelpers.applyCaseSensitive; +import static org.apache.gravitino.catalog.CapabilityHelpers.getCapability; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.connector.capability.Capability; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.rel.types.Type; + +/** + * {@code FunctionNormalizeDispatcher} normalizes function identifiers and namespaces by applying + * case-sensitivity and other naming capabilities before delegating to the underlying dispatcher. + */ +public class FunctionNormalizeDispatcher implements FunctionDispatcher { + private final CatalogManager catalogManager; + private final FunctionDispatcher dispatcher; + + public FunctionNormalizeDispatcher(FunctionDispatcher dispatcher, CatalogManager catalogManager) { + this.dispatcher = dispatcher; + this.catalogManager = catalogManager; + } + + @Override + public NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException { + Namespace caseSensitiveNs = normalizeCaseSensitive(namespace); + NameIdentifier[] identifiers = dispatcher.listFunctions(caseSensitiveNs); + return normalizeCaseSensitive(identifiers); + } + + @Override + public Function[] listFunctionInfos(Namespace namespace) throws NoSuchSchemaException { + return dispatcher.listFunctionInfos(normalizeCaseSensitive(namespace)); + } + + @Override + public Function getFunction(NameIdentifier ident) throws NoSuchFunctionException { + return dispatcher.getFunction(normalizeCaseSensitive(ident)); + } + + @Override + public Function getFunction(NameIdentifier ident, int version) + throws NoSuchFunctionException, NoSuchFunctionVersionException { + return dispatcher.getFunction(normalizeCaseSensitive(ident), version); + } + + @Override + public boolean functionExists(NameIdentifier ident) { + return dispatcher.functionExists(normalizeCaseSensitive(ident)); + } + + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + FunctionType functionType, + boolean deterministic, + Type returnType, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + return dispatcher.registerFunction( + normalizeNameIdentifier(ident), + comment, + functionType, + deterministic, + returnType, + definitions); + } + + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + boolean deterministic, + FunctionColumn[] returnColumns, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + return dispatcher.registerFunction( + normalizeNameIdentifier(ident), comment, deterministic, returnColumns, definitions); + } + + @Override + public Function alterFunction(NameIdentifier ident, FunctionChange... changes) + throws NoSuchFunctionException, IllegalArgumentException { + return dispatcher.alterFunction(normalizeCaseSensitive(ident), changes); + } + + @Override + public boolean dropFunction(NameIdentifier ident) { + return dispatcher.dropFunction(normalizeCaseSensitive(ident)); + } + + private Namespace normalizeCaseSensitive(Namespace namespace) { + Capability capabilities = getCapability(NameIdentifier.of(namespace.levels()), catalogManager); + return applyCaseSensitive(namespace, Capability.Scope.FUNCTION, capabilities); + } + + private NameIdentifier normalizeCaseSensitive(NameIdentifier functionIdent) { + Capability capabilities = getCapability(functionIdent, catalogManager); + return applyCaseSensitive(functionIdent, Capability.Scope.FUNCTION, capabilities); + } + + private NameIdentifier[] normalizeCaseSensitive(NameIdentifier[] functionIdents) { + if (ArrayUtils.isEmpty(functionIdents)) { + return functionIdents; + } + + Capability capabilities = getCapability(functionIdents[0], catalogManager); + return applyCaseSensitive(functionIdents, Capability.Scope.FUNCTION, capabilities); + } + + private NameIdentifier normalizeNameIdentifier(NameIdentifier functionIdent) { + Capability capability = getCapability(functionIdent, catalogManager); + return applyCapabilities(functionIdent, Capability.Scope.FUNCTION, capability); + } +} diff --git a/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java new file mode 100644 index 00000000000..2392053f69d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import com.google.common.base.Preconditions; +import java.util.Arrays; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.lock.LockType; +import org.apache.gravitino.lock.TreeLockUtils; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.storage.IdGenerator; + +/** + * {@code FunctionOperationDispatcher} is responsible for dispatching function-related operations. + * + *

Unlike ModelCatalog which manages its own schemas (via ManagedSchemaOperations), functions are + * registered under schemas managed by the underlying catalog (e.g., Hive, Iceberg). Therefore, this + * dispatcher validates schema existence by calling the underlying catalog's schema operations, then + * delegates actual storage operations to {@link ManagedFunctionOperations}. + */ +public class FunctionOperationDispatcher extends OperationDispatcher implements FunctionDispatcher { + + private final SchemaOperationDispatcher schemaOps; + private final ManagedFunctionOperations managedFunctionOps; + + /** + * Creates a new FunctionOperationDispatcher instance. + * + * @param catalogManager The CatalogManager instance to be used for function operations. + * @param store The EntityStore instance to be used for function operations. + * @param idGenerator The IdGenerator instance to be used for function operations. + */ + public FunctionOperationDispatcher( + CatalogManager catalogManager, + SchemaOperationDispatcher schemaOps, + EntityStore store, + IdGenerator idGenerator) { + super(catalogManager, store, idGenerator); + this.schemaOps = schemaOps; + this.managedFunctionOps = new ManagedFunctionOperations(store, idGenerator); + } + + /** + * List the functions in a namespace from the catalog. + * + * @param namespace A namespace. + * @return An array of function identifiers in the namespace. + * @throws NoSuchSchemaException If the schema does not exist. + */ + @Override + public NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException { + return Arrays.stream(listFunctionInfos(namespace)) + .map(f -> NameIdentifier.of(namespace, f.name())) + .toArray(NameIdentifier[]::new); + } + + @Override + public Function[] listFunctionInfos(Namespace namespace) throws NoSuchSchemaException { + NameIdentifier schemaIdent = NameIdentifier.of(namespace.levels()); + // Validate schema exists in the underlying catalog + schemaOps.loadSchema(schemaIdent); + + return TreeLockUtils.doWithTreeLock( + schemaIdent, LockType.READ, () -> managedFunctionOps.listFunctionInfos(namespace)); + } + + /** + * Get a function by {@link NameIdentifier} from the catalog. Returns the latest version. + * + * @param ident A function identifier. + * @return The latest version of the function with the given name. + * @throws NoSuchFunctionException If the function does not exist. + */ + @Override + public Function getFunction(NameIdentifier ident) throws NoSuchFunctionException { + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + // Validate schema exists in the underlying catalog + if (!schemaOps.schemaExists(schemaIdent)) { + throw new NoSuchFunctionException("Schema does not exist: %s", schemaIdent); + } + + return TreeLockUtils.doWithTreeLock( + ident, LockType.READ, () -> managedFunctionOps.getFunction(ident)); + } + + /** + * Get a function by {@link NameIdentifier} and version from the catalog. + * + * @param ident A function identifier. + * @param version The function version, counted from 0. + * @return The function with the given name and version. + * @throws NoSuchFunctionException If the function does not exist. + * @throws NoSuchFunctionVersionException If the function version does not exist. + */ + @Override + public Function getFunction(NameIdentifier ident, int version) + throws NoSuchFunctionException, NoSuchFunctionVersionException { + Preconditions.checkArgument(version >= 0, "Function version must be non-negative"); + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + // Validate schema exists in the underlying catalog + if (!schemaOps.schemaExists(schemaIdent)) { + throw new NoSuchFunctionException("Schema does not exist: %s", schemaIdent); + } + + return TreeLockUtils.doWithTreeLock( + ident, LockType.READ, () -> managedFunctionOps.getFunction(ident, version)); + } + + /** + * Register a scalar or aggregate function with one or more definitions (overloads). + * + * @param ident The function identifier. + * @param comment The optional function comment. + * @param functionType The function type. + * @param deterministic Whether the function is deterministic. + * @param returnType The return type. + * @param definitions The function definitions. + * @return The registered function. + * @throws NoSuchSchemaException If the schema does not exist. + * @throws FunctionAlreadyExistsException If the function already exists. + */ + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + FunctionType functionType, + boolean deterministic, + Type returnType, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + Preconditions.checkArgument( + functionType == FunctionType.SCALAR || functionType == FunctionType.AGGREGATE, + "This method is for scalar or aggregate functions only"); + Preconditions.checkArgument(returnType != null, "Return type is required"); + Preconditions.checkArgument( + definitions != null && definitions.length > 0, "At least one definition is required"); + + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + // Validate schema exists in the underlying catalog + schemaOps.loadSchema(schemaIdent); + + return TreeLockUtils.doWithTreeLock( + ident, + LockType.WRITE, + () -> + managedFunctionOps.registerFunction( + ident, comment, functionType, deterministic, returnType, definitions)); + } + + /** + * Register a table-valued function with one or more definitions (overloads). + * + * @param ident The function identifier. + * @param comment The optional function comment. + * @param deterministic Whether the function is deterministic. + * @param returnColumns The return columns. + * @param definitions The function definitions. + * @return The registered function. + * @throws NoSuchSchemaException If the schema does not exist. + * @throws FunctionAlreadyExistsException If the function already exists. + */ + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + boolean deterministic, + FunctionColumn[] returnColumns, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + Preconditions.checkArgument( + returnColumns != null && returnColumns.length > 0, + "At least one return column is required for table-valued function"); + Preconditions.checkArgument( + definitions != null && definitions.length > 0, "At least one definition is required"); + + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + // Validate schema exists in the underlying catalog + schemaOps.loadSchema(schemaIdent); + + return TreeLockUtils.doWithTreeLock( + ident, + LockType.WRITE, + () -> + managedFunctionOps.registerFunction( + ident, comment, deterministic, returnColumns, definitions)); + } + + /** + * Applies {@link FunctionChange changes} to a function in the catalog. + * + * @param ident the {@link NameIdentifier} instance of the function to alter. + * @param changes the several {@link FunctionChange} instances to apply to the function. + * @return the updated {@link Function} instance. + * @throws NoSuchFunctionException If the function does not exist. + * @throws IllegalArgumentException If the change is rejected by the implementation. + */ + @Override + public Function alterFunction(NameIdentifier ident, FunctionChange... changes) + throws NoSuchFunctionException, IllegalArgumentException { + Preconditions.checkArgument( + changes != null && changes.length > 0, "At least one change is required"); + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + if (!schemaOps.schemaExists(schemaIdent)) { + throw new NoSuchFunctionException("Schema does not exist: %s", schemaIdent); + } + + return TreeLockUtils.doWithTreeLock( + ident, LockType.WRITE, () -> managedFunctionOps.alterFunction(ident, changes)); + } + + /** + * Drop a function by name. + * + * @param ident The name identifier of the function. + * @return True if the function is deleted, false if the function does not exist. + */ + @Override + public boolean dropFunction(NameIdentifier ident) { + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + if (!schemaOps.schemaExists(schemaIdent)) { + return false; + } + + return TreeLockUtils.doWithTreeLock( + ident, LockType.WRITE, () -> managedFunctionOps.dropFunction(ident)); + } +} diff --git a/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java b/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java new file mode 100644 index 00000000000..f080f6ee8b3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionCatalog; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.storage.IdGenerator; + +/** + * {@code ManagedFunctionOperations} provides the storage-level operations for managing functions in + * Gravitino's EntityStore. + * + *

This class handles the actual persistence of function metadata, including: + * + *

+ */ +public class ManagedFunctionOperations implements FunctionCatalog { + + @SuppressWarnings("UnusedVariable") + private static final int INIT_VERSION = 0; + + @SuppressWarnings("UnusedVariable") + private final EntityStore store; + + @SuppressWarnings("UnusedVariable") + private final IdGenerator idGenerator; + + /** + * Creates a new ManagedFunctionOperations instance. + * + * @param store The EntityStore instance for function persistence. + * @param idGenerator The IdGenerator instance for generating unique IDs. + */ + public ManagedFunctionOperations(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + @Override + public NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException("listFunctions: FunctionEntity not yet implemented"); + } + + @Override + public Function[] listFunctionInfos(Namespace namespace) throws NoSuchSchemaException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException( + "listFunctionInfos: FunctionEntity not yet implemented"); + } + + @Override + public Function getFunction(NameIdentifier ident) throws NoSuchFunctionException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException("getFunction: FunctionEntity not yet implemented"); + } + + @Override + public Function getFunction(NameIdentifier ident, int version) + throws NoSuchFunctionException, NoSuchFunctionVersionException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException( + "getFunction with version: FunctionEntity not yet implemented"); + } + + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + FunctionType functionType, + boolean deterministic, + Type returnType, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException("registerFunction: FunctionEntity not yet implemented"); + } + + @Override + public Function registerFunction( + NameIdentifier ident, + String comment, + boolean deterministic, + FunctionColumn[] returnColumns, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException( + "registerFunction for table-valued functions: FunctionEntity not yet implemented"); + } + + @Override + public Function alterFunction(NameIdentifier ident, FunctionChange... changes) + throws NoSuchFunctionException, IllegalArgumentException { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException("alterFunction: FunctionEntity not yet implemented"); + } + + @Override + public boolean dropFunction(NameIdentifier ident) { + // TODO: Implement when FunctionEntity is available + throw new UnsupportedOperationException("dropFunction: FunctionEntity not yet implemented"); + } +} diff --git a/core/src/main/java/org/apache/gravitino/connector/capability/Capability.java b/core/src/main/java/org/apache/gravitino/connector/capability/Capability.java index 37d517f2d21..473f528b660 100644 --- a/core/src/main/java/org/apache/gravitino/connector/capability/Capability.java +++ b/core/src/main/java/org/apache/gravitino/connector/capability/Capability.java @@ -40,7 +40,8 @@ enum Scope { FILESET, TOPIC, PARTITION, - MODEL + MODEL, + FUNCTION } /** @@ -141,6 +142,10 @@ public CapabilityResult specificationOnName(Scope scope, String name) { @Override public CapabilityResult managedStorage(Scope scope) { + if (scope == Scope.FUNCTION) { + return CapabilityResult.SUPPORTED; + } + return CapabilityResult.unsupported( String.format("The %s entity is not fully managed by Gravitino.", scope)); } From 3c47d3243c1f4e26982ee0659fa84a4ffbb3643b Mon Sep 17 00:00:00 2001 From: mchades Date: Thu, 25 Dec 2025 19:14:12 +0800 Subject: [PATCH 5/6] add core udf operations --- .../java/org/apache/gravitino/Entity.java | 3 +- .../catalog/ManagedFunctionOperations.java | 513 +++++++++++- .../apache/gravitino/meta/FunctionEntity.java | 357 +++++++++ .../storage/relational/JDBCBackend.java | 2 + .../TestManagedFunctionOperations.java | 732 ++++++++++++++++++ 5 files changed, 1584 insertions(+), 23 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/meta/FunctionEntity.java create mode 100644 core/src/test/java/org/apache/gravitino/catalog/TestManagedFunctionOperations.java diff --git a/core/src/main/java/org/apache/gravitino/Entity.java b/core/src/main/java/org/apache/gravitino/Entity.java index fc575b72b12..2df8e7a8da6 100644 --- a/core/src/main/java/org/apache/gravitino/Entity.java +++ b/core/src/main/java/org/apache/gravitino/Entity.java @@ -80,7 +80,8 @@ enum EntityType { TABLE_STATISTIC, JOB_TEMPLATE, JOB, - AUDIT; + AUDIT, + FUNCTION; } /** diff --git a/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java b/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java index f080f6ee8b3..cc29ad5a4cb 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java +++ b/core/src/main/java/org/apache/gravitino/catalog/ManagedFunctionOperations.java @@ -18,10 +18,21 @@ */ package org.apache.gravitino.catalog; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchFunctionException; import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; import org.apache.gravitino.exceptions.NoSuchSchemaException; @@ -30,9 +41,16 @@ import org.apache.gravitino.function.FunctionChange; import org.apache.gravitino.function.FunctionColumn; import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionDefinitions; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionParam; import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.FunctionEntity; +import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.PrincipalUtils; /** * {@code ManagedFunctionOperations} provides the storage-level operations for managing functions in @@ -49,13 +67,10 @@ */ public class ManagedFunctionOperations implements FunctionCatalog { - @SuppressWarnings("UnusedVariable") private static final int INIT_VERSION = 0; - @SuppressWarnings("UnusedVariable") private final EntityStore store; - @SuppressWarnings("UnusedVariable") private final IdGenerator idGenerator; /** @@ -71,29 +86,56 @@ public ManagedFunctionOperations(EntityStore store, IdGenerator idGenerator) { @Override public NameIdentifier[] listFunctions(Namespace namespace) throws NoSuchSchemaException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException("listFunctions: FunctionEntity not yet implemented"); + return Arrays.stream(listFunctionInfos(namespace)) + .map(f -> NameIdentifier.of(namespace, f.name())) + .toArray(NameIdentifier[]::new); } @Override public Function[] listFunctionInfos(Namespace namespace) throws NoSuchSchemaException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException( - "listFunctionInfos: FunctionEntity not yet implemented"); + try { + List functions = + store.list(namespace, FunctionEntity.class, Entity.EntityType.FUNCTION); + return functions.toArray(FunctionEntity[]::new); + + } catch (NoSuchEntityException e) { + throw new NoSuchSchemaException(e, "Schema %s does not exist", namespace); + } catch (IOException e) { + throw new RuntimeException("Failed to list functions in namespace " + namespace, e); + } } @Override public Function getFunction(NameIdentifier ident) throws NoSuchFunctionException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException("getFunction: FunctionEntity not yet implemented"); + return getFunction(ident, FunctionEntity.LATEST_VERSION); } @Override public Function getFunction(NameIdentifier ident, int version) throws NoSuchFunctionException, NoSuchFunctionVersionException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException( - "getFunction with version: FunctionEntity not yet implemented"); + NameIdentifier versionedIdent = toVersionedIdent(ident, version); + try { + return store.get(versionedIdent, Entity.EntityType.FUNCTION, FunctionEntity.class); + + } catch (NoSuchEntityException e) { + if (version == FunctionEntity.LATEST_VERSION) { + throw new NoSuchFunctionException(e, "Function %s does not exist", ident); + } + // Check if the function exists at all + try { + store.get( + toVersionedIdent(ident, FunctionEntity.LATEST_VERSION), + Entity.EntityType.FUNCTION, + FunctionEntity.class); + // Function exists, but version doesn't + throw new NoSuchFunctionVersionException( + e, "Function %s version %d does not exist", ident, version); + } catch (NoSuchEntityException | IOException ex) { + throw new NoSuchFunctionException(e, "Function %s does not exist", ident); + } + } catch (IOException e) { + throw new RuntimeException("Failed to get function " + ident, e); + } } @Override @@ -105,8 +147,8 @@ public Function registerFunction( Type returnType, FunctionDefinition[] definitions) throws NoSuchSchemaException, FunctionAlreadyExistsException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException("registerFunction: FunctionEntity not yet implemented"); + return doRegisterFunction( + ident, comment, functionType, deterministic, returnType, null, definitions); } @Override @@ -117,21 +159,448 @@ public Function registerFunction( FunctionColumn[] returnColumns, FunctionDefinition[] definitions) throws NoSuchSchemaException, FunctionAlreadyExistsException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException( - "registerFunction for table-valued functions: FunctionEntity not yet implemented"); + return doRegisterFunction( + ident, comment, FunctionType.TABLE, deterministic, null, returnColumns, definitions); } @Override public Function alterFunction(NameIdentifier ident, FunctionChange... changes) throws NoSuchFunctionException, IllegalArgumentException { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException("alterFunction: FunctionEntity not yet implemented"); + try { + return store.update( + ident, + FunctionEntity.class, + Entity.EntityType.FUNCTION, + oldEntity -> applyChanges(oldEntity, changes)); + + } catch (NoSuchEntityException e) { + throw new NoSuchFunctionException(e, "Function %s does not exist", ident); + } catch (EntityAlreadyExistsException e) { + throw new IllegalArgumentException("Failed to alter function " + ident, e); + } catch (IOException e) { + throw new RuntimeException("Failed to alter function " + ident, e); + } } @Override public boolean dropFunction(NameIdentifier ident) { - // TODO: Implement when FunctionEntity is available - throw new UnsupportedOperationException("dropFunction: FunctionEntity not yet implemented"); + try { + return store.delete(ident, Entity.EntityType.FUNCTION); + } catch (NoSuchEntityException e) { + return false; + } catch (IOException e) { + throw new RuntimeException("Failed to drop function " + ident, e); + } + } + + /** + * Converts a function identifier to a versioned identifier. The versioned identifier uses the + * version number as the name to allow the store to retrieve specific versions. + * + * @param ident The function identifier. + * @param version The version number, or {@link FunctionEntity#LATEST_VERSION} for the latest. + * @return The versioned identifier. + */ + private NameIdentifier toVersionedIdent(NameIdentifier ident, int version) { + return NameIdentifier.of( + ident.namespace().level(0), + ident.namespace().level(1), + ident.namespace().level(2), + ident.name(), + String.valueOf(version)); + } + + private Function doRegisterFunction( + NameIdentifier ident, + String comment, + FunctionType functionType, + boolean deterministic, + Type returnType, + FunctionColumn[] returnColumns, + FunctionDefinition[] definitions) + throws NoSuchSchemaException, FunctionAlreadyExistsException { + Preconditions.checkArgument( + definitions != null && definitions.length > 0, + "At least one function definition must be provided"); + + // Validate definitions for arity overlap when there are multiple definitions + if (definitions.length > 1) { + validateDefinitionsNoArityOverlap(definitions); + } + + String currentUser = PrincipalUtils.getCurrentUserName(); + Instant now = Instant.now(); + AuditInfo auditInfo = AuditInfo.builder().withCreator(currentUser).withCreateTime(now).build(); + + FunctionEntity functionEntity = + FunctionEntity.builder() + .withId(idGenerator.nextId()) + .withName(ident.name()) + .withNamespace(ident.namespace()) + .withComment(comment) + .withFunctionType(functionType) + .withDeterministic(deterministic) + .withReturnType(returnType) + .withReturnColumns(returnColumns) + .withDefinitions(definitions) + .withVersion(INIT_VERSION) + .withAuditInfo(auditInfo) + .build(); + + try { + store.put(functionEntity, false /* overwrite */); + return functionEntity; + + } catch (NoSuchEntityException e) { + throw new NoSuchSchemaException(e, "Schema %s does not exist", ident.namespace()); + } catch (EntityAlreadyExistsException e) { + throw new FunctionAlreadyExistsException(e, "Function %s already exists", ident); + } catch (IOException e) { + throw new RuntimeException("Failed to register function " + ident, e); + } + } + + private FunctionEntity applyChanges(FunctionEntity oldEntity, FunctionChange... changes) { + String newComment = oldEntity.comment(); + List newDefinitions = + new ArrayList<>(Arrays.asList(oldEntity.definitions())); + + for (FunctionChange change : changes) { + if (change instanceof FunctionChange.UpdateComment) { + newComment = ((FunctionChange.UpdateComment) change).newComment(); + + } else if (change instanceof FunctionChange.AddDefinition) { + FunctionDefinition defToAdd = ((FunctionChange.AddDefinition) change).definition(); + validateNoArityOverlap(newDefinitions, defToAdd); + newDefinitions.add(defToAdd); + + } else if (change instanceof FunctionChange.RemoveDefinition) { + FunctionParam[] paramsToRemove = ((FunctionChange.RemoveDefinition) change).parameters(); + validateRemoveDefinition(newDefinitions, paramsToRemove); + newDefinitions.removeIf(def -> parametersMatch(def.parameters(), paramsToRemove)); + + } else if (change instanceof FunctionChange.AddImpl) { + FunctionChange.AddImpl addImpl = (FunctionChange.AddImpl) change; + FunctionParam[] targetParams = addImpl.parameters(); + FunctionImpl implToAdd = addImpl.implementation(); + newDefinitions = addImplToDefinition(newDefinitions, targetParams, implToAdd); + + } else if (change instanceof FunctionChange.UpdateImpl) { + FunctionChange.UpdateImpl updateImpl = (FunctionChange.UpdateImpl) change; + FunctionParam[] targetParams = updateImpl.parameters(); + FunctionImpl.RuntimeType runtime = updateImpl.runtime(); + FunctionImpl newImpl = updateImpl.implementation(); + newDefinitions = updateImplInDefinition(newDefinitions, targetParams, runtime, newImpl); + + } else if (change instanceof FunctionChange.RemoveImpl) { + FunctionChange.RemoveImpl removeImpl = (FunctionChange.RemoveImpl) change; + FunctionParam[] targetParams = removeImpl.parameters(); + FunctionImpl.RuntimeType runtime = removeImpl.runtime(); + newDefinitions = removeImplFromDefinition(newDefinitions, targetParams, runtime); + + } else { + throw new IllegalArgumentException("Unknown function change: " + change); + } + } + + String currentUser = PrincipalUtils.getCurrentUserName(); + Instant now = Instant.now(); + AuditInfo newAuditInfo = + AuditInfo.builder() + .withCreator(oldEntity.auditInfo().creator()) + .withCreateTime(oldEntity.auditInfo().createTime()) + .withLastModifier(currentUser) + .withLastModifiedTime(now) + .build(); + + return FunctionEntity.builder() + .withId(oldEntity.id()) + .withName(oldEntity.name()) + .withNamespace(oldEntity.namespace()) + .withComment(newComment) + .withFunctionType(oldEntity.functionType()) + .withDeterministic(oldEntity.deterministic()) + .withReturnType(oldEntity.returnType()) + .withReturnColumns(oldEntity.returnColumns()) + .withDefinitions(newDefinitions.toArray(new FunctionDefinition[0])) + .withVersion(oldEntity.version() + 1) + .withAuditInfo(newAuditInfo) + .build(); + } + + /** + * Validates that all definitions in the array do not have overlapping arities. This is used when + * registering a function with multiple definitions. + * + *

Gravitino enforces strict validation to prevent ambiguity. Operations MUST fail if any + * definition's invocation arities overlap with another. For example, if an existing definition + * {@code foo(int, float default 1.0)} supports arities {@code (int)} and {@code (int, float)}, + * adding a new definition {@code foo(int, string default 'x')} (which supports {@code (int)} and + * {@code (int, string)}) will be REJECTED because both support the call {@code foo(1)}. This + * ensures every function invocation deterministically maps to a single definition. + * + * @param definitions The array of definitions to validate. + * @throws IllegalArgumentException If any two definitions have overlapping arities. + */ + private void validateDefinitionsNoArityOverlap(FunctionDefinition[] definitions) { + for (int i = 0; i < definitions.length; i++) { + Set aritiesI = computeArities(definitions[i]); + for (int j = i + 1; j < definitions.length; j++) { + Set aritiesJ = computeArities(definitions[j]); + for (String arity : aritiesI) { + if (aritiesJ.contains(arity)) { + throw new IllegalArgumentException( + String.format( + "Cannot register function: definitions at index %d and %d have overlapping " + + "arity '%s'. This would create ambiguous function invocations.", + i, j, arity)); + } + } + } + } + } + + /** + * Validates that a new definition does not create ambiguous function arities with existing + * definitions. Each definition can support multiple arities based on parameters with default + * values. + * + *

Gravitino enforces strict validation to prevent ambiguity. Operations MUST fail if a new + * definition's invocation arities overlap with existing ones. For example, if an existing + * definition {@code foo(int, float default 1.0)} supports arities {@code (int)} and {@code (int, + * float)}, adding a new definition {@code foo(int, string default 'x')} (which supports {@code + * (int)} and {@code (int, string)}) will be REJECTED because both support the call {@code + * foo(1)}. This ensures every function invocation deterministically maps to a single definition. + * + * @param existingDefinitions The current definitions. + * @param newDefinition The definition to add. + * @throws IllegalArgumentException If the new definition creates overlapping arities. + */ + private void validateNoArityOverlap( + List existingDefinitions, FunctionDefinition newDefinition) { + Set newArities = computeArities(newDefinition); + + for (FunctionDefinition existing : existingDefinitions) { + Set existingArities = computeArities(existing); + for (String arity : newArities) { + if (existingArities.contains(arity)) { + throw new IllegalArgumentException( + String.format( + "Cannot add definition: arity '%s' overlaps with an existing definition. " + + "This would create ambiguous function invocations.", + arity)); + } + } + } + } + + /** + * Computes all possible invocation arities for a function definition. A definition with N + * parameters where the last M have default values supports arities from (N-M) to N parameters. + * + *

For example: + * + *

    + *
  • {@code foo(int a)} → arities: {@code ["int"]} + *
  • {@code foo(int a, float b)} → arities: {@code ["int,float"]} + *
  • {@code foo(int a, float b default 1.0)} → arities: {@code ["int", "int,float"]} + *
  • {@code foo(int a, float b default 1.0, string c default 'x')} → arities: {@code ["int", + * "int,float", "int,float,string"]} + *
  • {@code foo()} (no args) → arities: {@code [""]} + *
+ * + * @param definition The function definition. + * @return A set of arity signatures (e.g., "int", "int,float", ""). + */ + private Set computeArities(FunctionDefinition definition) { + Set arities = new HashSet<>(); + FunctionParam[] params = definition.parameters(); + + // Find the first parameter with a default value + int firstDefaultIndex = params.length; + for (int i = 0; i < params.length; i++) { + if (params[i].defaultValue() != Column.DEFAULT_VALUE_NOT_SET) { + firstDefaultIndex = i; + break; + } + } + + // Generate all possible arities from firstDefaultIndex to params.length + for (int i = firstDefaultIndex; i <= params.length; i++) { + StringBuilder arity = new StringBuilder(); + for (int j = 0; j < i; j++) { + if (j > 0) { + arity.append(","); + } + arity.append(params[j].dataType().simpleString()); + } + arities.add(arity.toString()); + } + + return arities; + } + + /** + * Validates that a definition can be removed. + * + * @param definitions The current definitions. + * @param paramsToRemove The parameters identifying the definition to remove. + * @throws IllegalArgumentException If the definition doesn't exist or is the only one. + */ + private void validateRemoveDefinition( + List definitions, FunctionParam[] paramsToRemove) { + boolean found = false; + for (FunctionDefinition def : definitions) { + if (parametersMatch(def.parameters(), paramsToRemove)) { + found = true; + break; + } + } + + if (!found) { + throw new IllegalArgumentException( + "Cannot remove definition: no definition found with the specified parameters"); + } + + if (definitions.size() == 1) { + throw new IllegalArgumentException( + "Cannot remove the only definition. Use dropFunction to remove the entire function."); + } + } + + private boolean parametersMatch(FunctionParam[] params1, FunctionParam[] params2) { + if (params1.length != params2.length) { + return false; + } + for (int i = 0; i < params1.length; i++) { + if (!params1[i].name().equals(params2[i].name()) + || !params1[i].dataType().equals(params2[i].dataType())) { + return false; + } + } + return true; + } + + private List addImplToDefinition( + List definitions, FunctionParam[] targetParams, FunctionImpl implToAdd) { + List result = new ArrayList<>(); + boolean found = false; + + for (FunctionDefinition def : definitions) { + if (parametersMatch(def.parameters(), targetParams)) { + found = true; + // Check if runtime already exists + for (FunctionImpl existingImpl : def.impls()) { + if (existingImpl.runtime() == implToAdd.runtime()) { + throw new IllegalArgumentException( + String.format( + "Cannot add implementation: runtime '%s' already exists in this definition. " + + "Use updateImpl to replace it.", + implToAdd.runtime())); + } + } + List impls = new ArrayList<>(Arrays.asList(def.impls())); + impls.add(implToAdd); + result.add(FunctionDefinitions.of(def.parameters(), impls.toArray(new FunctionImpl[0]))); + } else { + result.add(def); + } + } + + if (!found) { + throw new IllegalArgumentException( + "Cannot add implementation: no definition found with the specified parameters"); + } + + return result; + } + + private List updateImplInDefinition( + List definitions, + FunctionParam[] targetParams, + FunctionImpl.RuntimeType runtime, + FunctionImpl newImpl) { + List result = new ArrayList<>(); + boolean definitionFound = false; + boolean runtimeFound = false; + + for (FunctionDefinition def : definitions) { + if (parametersMatch(def.parameters(), targetParams)) { + definitionFound = true; + List impls = new ArrayList<>(); + for (FunctionImpl impl : def.impls()) { + if (impl.runtime() == runtime) { + runtimeFound = true; + impls.add(newImpl); + } else { + impls.add(impl); + } + } + result.add(FunctionDefinitions.of(def.parameters(), impls.toArray(new FunctionImpl[0]))); + } else { + result.add(def); + } + } + + if (!definitionFound) { + throw new IllegalArgumentException( + "Cannot update implementation: no definition found with the specified parameters"); + } + + if (!runtimeFound) { + throw new IllegalArgumentException( + String.format( + "Cannot update implementation: runtime '%s' not found in the definition", runtime)); + } + + return result; + } + + private List removeImplFromDefinition( + List definitions, + FunctionParam[] targetParams, + FunctionImpl.RuntimeType runtime) { + List result = new ArrayList<>(); + boolean definitionFound = false; + boolean runtimeFound = false; + + for (FunctionDefinition def : definitions) { + if (parametersMatch(def.parameters(), targetParams)) { + definitionFound = true; + + // Check if this is the only implementation + if (def.impls().length == 1) { + if (def.impls()[0].runtime() == runtime) { + throw new IllegalArgumentException( + "Cannot remove the only implementation. Use removeDefinition to remove the entire definition."); + } + } + + List impls = new ArrayList<>(); + for (FunctionImpl impl : def.impls()) { + if (impl.runtime() == runtime) { + runtimeFound = true; + } else { + impls.add(impl); + } + } + result.add(FunctionDefinitions.of(def.parameters(), impls.toArray(new FunctionImpl[0]))); + } else { + result.add(def); + } + } + + if (!definitionFound) { + throw new IllegalArgumentException( + "Cannot remove implementation: no definition found with the specified parameters"); + } + + if (!runtimeFound) { + throw new IllegalArgumentException( + String.format( + "Cannot remove implementation: runtime '%s' not found in the definition", runtime)); + } + + return result; } } diff --git a/core/src/main/java/org/apache/gravitino/meta/FunctionEntity.java b/core/src/main/java/org/apache/gravitino/meta/FunctionEntity.java new file mode 100644 index 00000000000..55a2288c087 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/meta/FunctionEntity.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.meta; + +import com.google.common.collect.Maps; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import lombok.ToString; +import org.apache.gravitino.Auditable; +import org.apache.gravitino.Entity; +import org.apache.gravitino.Field; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.rel.types.Type; + +/** + * A class representing a function entity in the metadata store. + * + *

This entity stores both the function metadata and its version information together, avoiding + * the need for separate FunctionEntity and FunctionVersionEntity. When retrieving, if version is + * set to the special value {@link #LATEST_VERSION}, the store should return the latest version. + */ +@ToString +public class FunctionEntity implements Entity, Auditable, HasIdentifier, Function { + + /** Special version value indicating the latest version should be retrieved. */ + public static final int LATEST_VERSION = -1; + + public static final Field ID = + Field.required("id", Long.class, "The unique id of the function entity."); + public static final Field NAME = + Field.required("name", String.class, "The name of the function entity."); + public static final Field COMMENT = + Field.optional("comment", String.class, "The comment or description of the function entity."); + public static final Field FUNCTION_TYPE = + Field.required("function_type", FunctionType.class, "The type of the function."); + public static final Field DETERMINISTIC = + Field.required("deterministic", Boolean.class, "Whether the function is deterministic."); + public static final Field RETURN_TYPE = + Field.optional( + "return_type", Type.class, "The return type for scalar or aggregate functions."); + public static final Field RETURN_COLUMNS = + Field.optional( + "return_columns", + FunctionColumn[].class, + "The output columns for table-valued functions."); + public static final Field DEFINITIONS = + Field.required("definitions", FunctionDefinition[].class, "The definitions of the function."); + public static final Field VERSION = + Field.required("version", Integer.class, "The version of the function entity."); + public static final Field AUDIT_INFO = + Field.required("audit_info", AuditInfo.class, "The audit details of the function entity."); + + private Long id; + private String name; + private Namespace namespace; + private String comment; + private FunctionType functionType; + private boolean deterministic; + private Type returnType; + private FunctionColumn[] returnColumns; + private FunctionDefinition[] definitions; + private Integer version; + private AuditInfo auditInfo; + + private FunctionEntity() {} + + @Override + public Map fields() { + Map fields = Maps.newHashMap(); + fields.put(ID, id); + fields.put(NAME, name); + fields.put(COMMENT, comment); + fields.put(FUNCTION_TYPE, functionType); + fields.put(DETERMINISTIC, deterministic); + fields.put(RETURN_TYPE, returnType); + fields.put(RETURN_COLUMNS, returnColumns); + fields.put(DEFINITIONS, definitions); + fields.put(VERSION, version); + fields.put(AUDIT_INFO, auditInfo); + + return Collections.unmodifiableMap(fields); + } + + @Override + public String name() { + return name; + } + + @Override + public Long id() { + return id; + } + + @Override + public Namespace namespace() { + return namespace; + } + + @Override + public String comment() { + return comment; + } + + @Override + public FunctionType functionType() { + return functionType; + } + + @Override + public boolean deterministic() { + return deterministic; + } + + @Override + public Type returnType() { + return returnType; + } + + @Override + public FunctionColumn[] returnColumns() { + return returnColumns != null ? returnColumns : new FunctionColumn[0]; + } + + @Override + public FunctionDefinition[] definitions() { + return definitions; + } + + @Override + public int version() { + return version; + } + + @Override + public AuditInfo auditInfo() { + return auditInfo; + } + + @Override + public EntityType type() { + return EntityType.FUNCTION; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof FunctionEntity)) { + return false; + } + + FunctionEntity that = (FunctionEntity) o; + return Objects.equals(id, that.id) + && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) + && Objects.equals(comment, that.comment) + && functionType == that.functionType + && deterministic == that.deterministic + && Objects.equals(returnType, that.returnType) + && Arrays.equals(returnColumns, that.returnColumns) + && Arrays.equals(definitions, that.definitions) + && Objects.equals(version, that.version) + && Objects.equals(auditInfo, that.auditInfo); + } + + @Override + public int hashCode() { + int result = + Objects.hash( + id, + name, + namespace, + comment, + functionType, + deterministic, + returnType, + version, + auditInfo); + result = 31 * result + Arrays.hashCode(returnColumns); + result = 31 * result + Arrays.hashCode(definitions); + return result; + } + + /** + * Creates a new builder for constructing a FunctionEntity. + * + * @return A new builder instance. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder class for creating instances of {@link FunctionEntity}. */ + public static class Builder { + private final FunctionEntity functionEntity; + + private Builder() { + functionEntity = new FunctionEntity(); + } + + /** + * Sets the unique id of the function entity. + * + * @param id The unique id. + * @return This builder instance. + */ + public Builder withId(Long id) { + functionEntity.id = id; + return this; + } + + /** + * Sets the name of the function entity. + * + * @param name The name of the function. + * @return This builder instance. + */ + public Builder withName(String name) { + functionEntity.name = name; + return this; + } + + /** + * Sets the namespace of the function entity. + * + * @param namespace The namespace. + * @return This builder instance. + */ + public Builder withNamespace(Namespace namespace) { + functionEntity.namespace = namespace; + return this; + } + + /** + * Sets the comment of the function entity. + * + * @param comment The comment or description. + * @return This builder instance. + */ + public Builder withComment(String comment) { + functionEntity.comment = comment; + return this; + } + + /** + * Sets the function type. + * + * @param functionType The type of the function (SCALAR, AGGREGATE, or TABLE). + * @return This builder instance. + */ + public Builder withFunctionType(FunctionType functionType) { + functionEntity.functionType = functionType; + return this; + } + + /** + * Sets whether the function is deterministic. + * + * @param deterministic True if the function is deterministic, false otherwise. + * @return This builder instance. + */ + public Builder withDeterministic(boolean deterministic) { + functionEntity.deterministic = deterministic; + return this; + } + + /** + * Sets the return type for scalar or aggregate functions. + * + * @param returnType The return type. + * @return This builder instance. + */ + public Builder withReturnType(Type returnType) { + functionEntity.returnType = returnType; + return this; + } + + /** + * Sets the return columns for table-valued functions. + * + * @param returnColumns The output columns. + * @return This builder instance. + */ + public Builder withReturnColumns(FunctionColumn[] returnColumns) { + functionEntity.returnColumns = returnColumns; + return this; + } + + /** + * Sets the function definitions. + * + * @param definitions The definitions (overloads) of the function. + * @return This builder instance. + */ + public Builder withDefinitions(FunctionDefinition[] definitions) { + functionEntity.definitions = definitions; + return this; + } + + /** + * Sets the version of the function entity. + * + * @param version The version number. + * @return This builder instance. + */ + public Builder withVersion(Integer version) { + functionEntity.version = version; + return this; + } + + /** + * Sets the audit information. + * + * @param auditInfo The audit information. + * @return This builder instance. + */ + public Builder withAuditInfo(AuditInfo auditInfo) { + functionEntity.auditInfo = auditInfo; + return this; + } + + /** + * Builds the FunctionEntity instance. + * + * @return The constructed FunctionEntity. + */ + public FunctionEntity build() { + functionEntity.validate(); + return functionEntity; + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 206d9521ff2..fee9d4f16fe 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -397,6 +397,7 @@ public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimelin return JobMetaService.getInstance() .deleteJobsByLegacyTimeline(legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case AUDIT: + case FUNCTION: return 0; // TODO: Implement hard delete logic for these entity types. @@ -426,6 +427,7 @@ public int deleteOldVersionData(Entity.EntityType entityType, long versionRetent case TABLE_STATISTIC: case JOB_TEMPLATE: case JOB: + case FUNCTION: // todo: remove once function versioning is supported // These entity types have not implemented multi-versions, so we can skip. return 0; diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestManagedFunctionOperations.java b/core/src/test/java/org/apache/gravitino/catalog/TestManagedFunctionOperations.java new file mode 100644 index 00000000000..5939272d117 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/catalog/TestManagedFunctionOperations.java @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionDefinitions; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; +import org.apache.gravitino.function.FunctionParam; +import org.apache.gravitino.function.FunctionParams; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.meta.FunctionEntity; +import org.apache.gravitino.rel.expressions.literals.Literals; +import org.apache.gravitino.rel.types.Types; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.storage.RandomIdGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestManagedFunctionOperations { + + private static final String METALAKE_NAME = "test_metalake"; + private static final String CATALOG_NAME = "test_catalog"; + private static final String SCHEMA_NAME = "schema1"; + + private final IdGenerator idGenerator = new RandomIdGenerator(); + private final Map entityMap = new HashMap<>(); + + private EntityStore store; + private ManagedFunctionOperations functionOperations; + + @BeforeEach + public void setUp() throws Exception { + entityMap.clear(); + store = createMockEntityStore(); + functionOperations = new ManagedFunctionOperations(store, idGenerator); + } + + @Test + public void testRegisterAndListFunctions() { + NameIdentifier func1Ident = getFunctionIdent("func1"); + FunctionParam[] params1 = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions1 = new FunctionDefinition[] {createSimpleDefinition(params1)}; + + functionOperations.registerFunction( + func1Ident, + "Test function 1", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions1); + + NameIdentifier func2Ident = getFunctionIdent("func2"); + FunctionParam[] params2 = + new FunctionParam[] { + FunctionParams.of("x", Types.StringType.get()), + FunctionParams.of("y", Types.StringType.get()) + }; + FunctionDefinition[] definitions2 = new FunctionDefinition[] {createSimpleDefinition(params2)}; + + functionOperations.registerFunction( + func2Ident, + "Test function 2", + FunctionType.SCALAR, + false, + Types.IntegerType.get(), + definitions2); + + // List functions + NameIdentifier[] functionIdents = functionOperations.listFunctions(getFunctionNamespace()); + Assertions.assertEquals(2, functionIdents.length); + Set functionNames = + Arrays.stream(functionIdents).map(NameIdentifier::name).collect(Collectors.toSet()); + + Assertions.assertTrue(functionNames.contains("func1")); + Assertions.assertTrue(functionNames.contains("func2")); + } + + @Test + public void testRegisterAndGetFunction() { + NameIdentifier funcIdent = getFunctionIdent("my_func"); + FunctionParam[] params = + new FunctionParam[] {FunctionParams.of("input", Types.StringType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + org.apache.gravitino.function.Function newFunc = + functionOperations.registerFunction( + funcIdent, + "My test function", + FunctionType.SCALAR, + true, + Types.IntegerType.get(), + definitions); + + Assertions.assertEquals("my_func", newFunc.name()); + Assertions.assertEquals("My test function", newFunc.comment()); + Assertions.assertEquals(FunctionType.SCALAR, newFunc.functionType()); + Assertions.assertTrue(newFunc.deterministic()); + Assertions.assertEquals(Types.IntegerType.get(), newFunc.returnType()); + Assertions.assertEquals(0, newFunc.version()); + + // Get function (latest version) + Function loadedFunc = functionOperations.getFunction(funcIdent); + Assertions.assertEquals(newFunc.name(), loadedFunc.name()); + Assertions.assertEquals(newFunc.comment(), loadedFunc.comment()); + + // Test register function that already exists + Assertions.assertThrows( + FunctionAlreadyExistsException.class, + () -> + functionOperations.registerFunction( + funcIdent, + "Another function", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions)); + + // Test get non-existing function + NameIdentifier nonExistingIdent = getFunctionIdent("non_existing_func"); + Assertions.assertThrows( + NoSuchFunctionException.class, () -> functionOperations.getFunction(nonExistingIdent)); + } + + @Test + public void testRegisterAndDropFunction() { + NameIdentifier funcIdent = getFunctionIdent("func_to_drop"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + functionOperations.registerFunction( + funcIdent, + "Function to drop", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions); + + // Drop the function + boolean dropped = functionOperations.dropFunction(funcIdent); + Assertions.assertTrue(dropped); + + // Verify the function is dropped + Assertions.assertThrows( + NoSuchFunctionException.class, () -> functionOperations.getFunction(funcIdent)); + + // Test drop non-existing function + Assertions.assertFalse(functionOperations.dropFunction(funcIdent)); + } + + @Test + public void testAlterFunctionUpdateComment() { + NameIdentifier funcIdent = getFunctionIdent("func_to_alter"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + functionOperations.registerFunction( + funcIdent, + "Original comment", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions); + + // Update comment + String newComment = "Updated comment"; + org.apache.gravitino.function.Function updatedFunc = + functionOperations.alterFunction(funcIdent, FunctionChange.updateComment(newComment)); + + Assertions.assertEquals(newComment, updatedFunc.comment()); + Assertions.assertEquals(1, updatedFunc.version()); + + // Verify the change is persisted + org.apache.gravitino.function.Function loadedFunc = functionOperations.getFunction(funcIdent); + Assertions.assertEquals(newComment, loadedFunc.comment()); + Assertions.assertEquals(1, loadedFunc.version()); + } + + @Test + public void testAlterFunctionAddDefinition() { + NameIdentifier funcIdent = getFunctionIdent("func_add_def"); + FunctionParam[] params1 = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions1 = new FunctionDefinition[] {createSimpleDefinition(params1)}; + + functionOperations.registerFunction( + funcIdent, + "Test function", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions1); + + // Add a new definition with different parameters + FunctionParam[] params2 = + new FunctionParam[] { + FunctionParams.of("a", Types.IntegerType.get()), + FunctionParams.of("b", Types.StringType.get()) + }; + FunctionDefinition newDef = createSimpleDefinition(params2); + + org.apache.gravitino.function.Function updatedFunc = + functionOperations.alterFunction(funcIdent, FunctionChange.addDefinition(newDef)); + + Assertions.assertEquals(2, updatedFunc.definitions().length); + Assertions.assertEquals(1, updatedFunc.version()); + } + + @Test + public void testAlterFunctionAddDefinitionWithOverlappingArity() { + NameIdentifier funcIdent = getFunctionIdent("func_overlap"); + + // Create definition: foo(int, float default 1.0) supports arities (int) and (int, float) + FunctionParam[] params1 = + new FunctionParam[] { + FunctionParams.of("a", Types.IntegerType.get()), + FunctionParams.of("b", Types.FloatType.get(), null, Literals.floatLiteral(1.0f)) + }; + FunctionDefinition[] definitions1 = new FunctionDefinition[] {createSimpleDefinition(params1)}; + + functionOperations.registerFunction( + funcIdent, + "Test function", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions1); + + // Try to add definition: foo(int, string default 'x') which supports (int) and (int, string) + // This should fail because both support the call foo(int) + FunctionParam[] params2 = + new FunctionParam[] { + FunctionParams.of("a", Types.IntegerType.get()), + FunctionParams.of("c", Types.StringType.get(), null, Literals.stringLiteral("x")) + }; + FunctionDefinition newDef = createSimpleDefinition(params2); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> functionOperations.alterFunction(funcIdent, FunctionChange.addDefinition(newDef))); + } + + @Test + public void testRegisterFunctionWithOverlappingDefinitions() { + NameIdentifier funcIdent = getFunctionIdent("func_overlap_register"); + + // Try to register with two definitions that have overlapping arities + FunctionParam[] params1 = + new FunctionParam[] { + FunctionParams.of("a", Types.IntegerType.get()), + FunctionParams.of("b", Types.FloatType.get(), null, Literals.floatLiteral(1.0f)) + }; + FunctionParam[] params2 = + new FunctionParam[] { + FunctionParams.of("a", Types.IntegerType.get()), + FunctionParams.of("c", Types.StringType.get(), null, Literals.stringLiteral("x")) + }; + + FunctionDefinition[] definitions = + new FunctionDefinition[] {createSimpleDefinition(params1), createSimpleDefinition(params2)}; + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.registerFunction( + funcIdent, + "Test function", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions)); + } + + @Test + public void testAlterFunctionRemoveDefinition() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_def"); + + // Create function with two definitions + FunctionParam[] params1 = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionParam[] params2 = new FunctionParam[] {FunctionParams.of("b", Types.StringType.get())}; + FunctionDefinition[] definitions = + new FunctionDefinition[] {createSimpleDefinition(params1), createSimpleDefinition(params2)}; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Remove one definition + Function updatedFunc = + functionOperations.alterFunction(funcIdent, FunctionChange.removeDefinition(params1)); + + Assertions.assertEquals(1, updatedFunc.definitions().length); + Assertions.assertEquals(1, updatedFunc.version()); + } + + @Test + public void testAlterFunctionRemoveOnlyDefinition() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_only"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to remove the only definition - should fail + Assertions.assertThrows( + IllegalArgumentException.class, + () -> functionOperations.alterFunction(funcIdent, FunctionChange.removeDefinition(params))); + } + + @Test + public void testAlterFunctionRemoveNonExistingDefinition() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_nonexist"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to remove a definition that doesn't exist + FunctionParam[] nonExistingParams = + new FunctionParam[] {FunctionParams.of("x", Types.StringType.get())}; + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, FunctionChange.removeDefinition(nonExistingParams))); + } + + @Test + public void testAlterFunctionAddImpl() { + NameIdentifier funcIdent = getFunctionIdent("func_add_impl"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Add Trino implementation + FunctionImpl trinoImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.TRINO, "com.example.TrinoUDF"); + org.apache.gravitino.function.Function updatedFunc = + functionOperations.alterFunction(funcIdent, FunctionChange.addImpl(params, trinoImpl)); + + Assertions.assertEquals(2, updatedFunc.definitions()[0].impls().length); + Assertions.assertEquals(1, updatedFunc.version()); + } + + @Test + public void testAlterFunctionAddImplDuplicateRuntime() { + NameIdentifier funcIdent = getFunctionIdent("func_add_impl_dup"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to add another Spark implementation - should fail + FunctionImpl anotherSparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.AnotherSparkUDF"); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, FunctionChange.addImpl(params, anotherSparkImpl))); + } + + @Test + public void testAlterFunctionAddImplToNonExistingDefinition() { + NameIdentifier funcIdent = getFunctionIdent("func_add_impl_nodef"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to add impl to non-existing definition + FunctionParam[] nonExistingParams = + new FunctionParam[] {FunctionParams.of("x", Types.StringType.get())}; + FunctionImpl impl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.TRINO, "com.example.TrinoUDF"); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, FunctionChange.addImpl(nonExistingParams, impl))); + } + + @Test + public void testAlterFunctionUpdateImpl() { + NameIdentifier funcIdent = getFunctionIdent("func_update_impl"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.OldSparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Update Spark implementation + FunctionImpl newSparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.NewSparkUDF"); + org.apache.gravitino.function.Function updatedFunc = + functionOperations.alterFunction( + funcIdent, + FunctionChange.updateImpl(params, FunctionImpl.RuntimeType.SPARK, newSparkImpl)); + + Assertions.assertEquals(1, updatedFunc.definitions()[0].impls().length); + Assertions.assertEquals(1, updatedFunc.version()); + } + + @Test + public void testAlterFunctionUpdateImplNonExistingRuntime() { + NameIdentifier funcIdent = getFunctionIdent("func_update_impl_noruntime"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to update Trino implementation which doesn't exist + FunctionImpl trinoImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.TRINO, "com.example.TrinoUDF"); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, + FunctionChange.updateImpl(params, FunctionImpl.RuntimeType.TRINO, trinoImpl))); + } + + @Test + public void testAlterFunctionRemoveImpl() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_impl"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionImpl trinoImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.TRINO, "com.example.TrinoUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl, trinoImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Remove Spark implementation + org.apache.gravitino.function.Function updatedFunc = + functionOperations.alterFunction( + funcIdent, FunctionChange.removeImpl(params, FunctionImpl.RuntimeType.SPARK)); + + Assertions.assertEquals(1, updatedFunc.definitions()[0].impls().length); + Assertions.assertEquals( + FunctionImpl.RuntimeType.TRINO, updatedFunc.definitions()[0].impls()[0].runtime()); + Assertions.assertEquals(1, updatedFunc.version()); + } + + @Test + public void testAlterFunctionRemoveOnlyImpl() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_only_impl"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to remove the only implementation - should fail + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, FunctionChange.removeImpl(params, FunctionImpl.RuntimeType.SPARK))); + } + + @Test + public void testAlterFunctionRemoveImplNonExistingRuntime() { + NameIdentifier funcIdent = getFunctionIdent("func_remove_impl_noruntime"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionImpl sparkImpl = + FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.SparkUDF"); + FunctionDefinition[] definitions = + new FunctionDefinition[] { + createDefinitionWithImpls(params, new FunctionImpl[] {sparkImpl}) + }; + + functionOperations.registerFunction( + funcIdent, "Test function", FunctionType.SCALAR, true, Types.StringType.get(), definitions); + + // Try to remove Trino implementation which doesn't exist + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + functionOperations.alterFunction( + funcIdent, FunctionChange.removeImpl(params, FunctionImpl.RuntimeType.TRINO))); + } + + @Test + public void testAlterNonExistingFunction() { + NameIdentifier nonExistingIdent = getFunctionIdent("non_existing_func"); + + Assertions.assertThrows( + NoSuchFunctionException.class, + () -> + functionOperations.alterFunction( + nonExistingIdent, FunctionChange.updateComment("new comment"))); + } + + @Test + public void testVersionIncrement() { + NameIdentifier funcIdent = getFunctionIdent("func_version"); + FunctionParam[] params = new FunctionParam[] {FunctionParams.of("a", Types.IntegerType.get())}; + FunctionDefinition[] definitions = new FunctionDefinition[] {createSimpleDefinition(params)}; + + org.apache.gravitino.function.Function v0 = + functionOperations.registerFunction( + funcIdent, + "v0 comment", + FunctionType.SCALAR, + true, + Types.StringType.get(), + definitions); + Assertions.assertEquals(0, v0.version()); + + org.apache.gravitino.function.Function v1 = + functionOperations.alterFunction(funcIdent, FunctionChange.updateComment("v1 comment")); + Assertions.assertEquals(1, v1.version()); + + org.apache.gravitino.function.Function v2 = + functionOperations.alterFunction(funcIdent, FunctionChange.updateComment("v2 comment")); + Assertions.assertEquals(2, v2.version()); + + // Get function should return latest version + org.apache.gravitino.function.Function latest = functionOperations.getFunction(funcIdent); + Assertions.assertEquals(2, latest.version()); + Assertions.assertEquals("v2 comment", latest.comment()); + } + + @SuppressWarnings("unchecked") + private EntityStore createMockEntityStore() throws Exception { + EntityStore mockStore = mock(EntityStore.class); + + // Mock put operation + doAnswer( + invocation -> { + FunctionEntity entity = invocation.getArgument(0); + boolean overwrite = invocation.getArgument(1); + NameIdentifier ident = entity.nameIdentifier(); + + if (!overwrite && entityMap.containsKey(ident)) { + throw new EntityAlreadyExistsException("Entity %s already exists", ident); + } + entityMap.put(ident, entity); + return null; + }) + .when(mockStore) + .put(any(FunctionEntity.class), any(Boolean.class)); + + // Mock get operation + when(mockStore.get( + any(NameIdentifier.class), eq(Entity.EntityType.FUNCTION), eq(FunctionEntity.class))) + .thenAnswer( + invocation -> { + NameIdentifier ident = invocation.getArgument(0); + FunctionEntity entity = findEntityByIdent(ident); + if (entity == null) { + throw new NoSuchEntityException("Entity %s does not exist", ident); + } + return entity; + }); + + // Mock update operation + when(mockStore.update( + any(NameIdentifier.class), + eq(FunctionEntity.class), + eq(Entity.EntityType.FUNCTION), + any(java.util.function.Function.class))) + .thenAnswer( + invocation -> { + NameIdentifier ident = invocation.getArgument(0); + java.util.function.Function updater = + invocation.getArgument(3); + + FunctionEntity oldEntity = findEntityByIdent(ident); + if (oldEntity == null) { + throw new NoSuchEntityException("Entity %s does not exist", ident); + } + + FunctionEntity newEntity = updater.apply(oldEntity); + NameIdentifier originalIdent = oldEntity.nameIdentifier(); + entityMap.put(originalIdent, newEntity); + return newEntity; + }); + + // Mock delete operation (2 parameters - default method that calls 3-parameter version) + when(mockStore.delete(any(NameIdentifier.class), eq(Entity.EntityType.FUNCTION))) + .thenAnswer( + invocation -> { + NameIdentifier ident = invocation.getArgument(0); + FunctionEntity entity = findEntityByIdent(ident); + if (entity == null) { + return false; + } + entityMap.remove(entity.nameIdentifier()); + return true; + }); + + // Mock list operation + when(mockStore.list( + any(Namespace.class), eq(FunctionEntity.class), eq(Entity.EntityType.FUNCTION))) + .thenAnswer( + invocation -> { + Namespace namespace = invocation.getArgument(0); + return entityMap.values().stream() + .filter(e -> e.namespace().equals(namespace)) + .collect(Collectors.toList()); + }); + + return mockStore; + } + + /** + * Finds an entity by identifier. This method handles both versioned identifiers (used by + * getFunction) and original identifiers (used by alterFunction and dropFunction). + * + *

Versioned identifier format: namespace = original_namespace + function_name, name = version + * Original identifier format: namespace = schema_namespace, name = function_name + */ + private FunctionEntity findEntityByIdent(NameIdentifier ident) { + // First, try to find by original identifier (direct match) + FunctionEntity directMatch = entityMap.get(ident); + if (directMatch != null) { + return directMatch; + } + + // If not found, try to interpret as versioned identifier + String[] levels = ident.namespace().levels(); + if (levels.length < 1) { + return null; + } + String functionName = levels[levels.length - 1]; + Namespace originalNamespace = Namespace.of(Arrays.copyOf(levels, levels.length - 1)); + + for (FunctionEntity entity : entityMap.values()) { + if (entity.name().equals(functionName) && entity.namespace().equals(originalNamespace)) { + return entity; + } + } + return null; + } + + private Namespace getFunctionNamespace() { + return Namespace.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME); + } + + private NameIdentifier getFunctionIdent(String functionName) { + return NameIdentifier.of(getFunctionNamespace(), functionName); + } + + private FunctionDefinition createSimpleDefinition(FunctionParam[] params) { + FunctionImpl impl = FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "com.example.TestUDF"); + return FunctionDefinitions.of(params, new FunctionImpl[] {impl}); + } + + private FunctionDefinition createDefinitionWithImpls( + FunctionParam[] params, FunctionImpl[] impls) { + return FunctionDefinitions.of(params, impls); + } +} From 0b4f14cf42d412f4022e633e46a52e3eb6446cae Mon Sep 17 00:00:00 2001 From: mchades Date: Fri, 26 Dec 2025 09:22:00 +0800 Subject: [PATCH 6/6] [#9529] feat(server): Add server-side REST interface for UDFs This PR implements the server-side REST API for User-Defined Functions (UDFs). Changes include: - Add FunctionDTO and related DTO classes for function metadata - Add FunctionRegisterRequest and FunctionUpdateRequest for API requests - Add FunctionResponse for API responses - Add FunctionOperations REST endpoint with register, get, update, delete operations - Add exception handlers for function-related errors - Add comprehensive unit tests for all new components --- .../dto/function/FunctionColumnDTO.java | 146 ++++ .../gravitino/dto/function/FunctionDTO.java | 313 +++++++ .../dto/function/FunctionDefinitionDTO.java | 163 ++++ .../dto/function/FunctionImplDTO.java | 115 +++ .../dto/function/FunctionParamDTO.java | 200 +++++ .../dto/function/FunctionResourcesDTO.java | 140 +++ .../gravitino/dto/function/JavaImplDTO.java | 68 ++ .../gravitino/dto/function/PythonImplDTO.java | 75 ++ .../gravitino/dto/function/SQLImplDTO.java | 67 ++ .../dto/requests/FunctionRegisterRequest.java | 98 +++ .../dto/requests/FunctionUpdateRequest.java | 253 ++++++ .../dto/requests/FunctionUpdatesRequest.java | 49 ++ .../dto/responses/FunctionListResponse.java | 57 ++ .../dto/responses/FunctionResponse.java | 70 ++ .../gravitino/dto/util/DTOConverters.java | 40 + .../dto/function/TestFunctionDTO.java | 263 ++++++ .../gravitino/utils/NameIdentifierUtil.java | 15 + .../apache/gravitino/utils/NamespaceUtil.java | 12 + .../gravitino/server/GravitinoServer.java | 2 + .../filter/GravitinoInterceptionService.java | 2 + .../server/web/rest/ExceptionHandlers.java | 42 + .../server/web/rest/FunctionOperations.java | 293 +++++++ .../web/rest/TestFunctionOperations.java | 804 ++++++++++++++++++ 23 files changed, 3287 insertions(+) create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionColumnDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionDefinitionDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionImplDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionParamDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/FunctionResourcesDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/JavaImplDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/PythonImplDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/function/SQLImplDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/FunctionRegisterRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdateRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdatesRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/FunctionListResponse.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/FunctionResponse.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/function/TestFunctionDTO.java create mode 100644 server/src/main/java/org/apache/gravitino/server/web/rest/FunctionOperations.java create mode 100644 server/src/test/java/org/apache/gravitino/server/web/rest/TestFunctionOperations.java diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionColumnDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionColumnDTO.java new file mode 100644 index 00000000000..e1b3590769e --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionColumnDTO.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import javax.annotation.Nullable; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.rel.types.Type; + +/** DTO for function column. */ +@Getter +@EqualsAndHashCode +public class FunctionColumnDTO { + + @JsonProperty("name") + private String name; + + @JsonProperty("dataType") + @JsonSerialize(using = JsonUtils.TypeSerializer.class) + @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) + private Type dataType; + + @Nullable + @JsonProperty("comment") + private String comment; + + private FunctionColumnDTO() {} + + private FunctionColumnDTO(String name, Type dataType, String comment) { + this.name = name; + this.dataType = dataType; + this.comment = comment; + } + + /** + * Convert this DTO to a {@link FunctionColumn} instance. + * + * @return The function column. + */ + public FunctionColumn toFunctionColumn() { + return FunctionColumn.of(name, dataType, comment); + } + + /** + * Create a {@link FunctionColumnDTO} from a {@link FunctionColumn} instance. + * + * @param column The function column. + * @return The function column DTO. + */ + public static FunctionColumnDTO fromFunctionColumn(FunctionColumn column) { + return new FunctionColumnDTO(column.name(), column.dataType(), column.comment()); + } + + @Override + public String toString() { + return "FunctionColumnDTO{" + + "name='" + + name + + '\'' + + ", dataType=" + + dataType + + ", comment='" + + comment + + '\'' + + '}'; + } + + /** Builder for {@link FunctionColumnDTO}. */ + public static class Builder { + private String name; + private Type dataType; + private String comment; + + /** + * Set the column name. + * + * @param name The column name. + * @return This builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Set the column data type. + * + * @param dataType The column data type. + * @return This builder. + */ + public Builder withDataType(Type dataType) { + this.dataType = dataType; + return this; + } + + /** + * Set the column comment. + * + * @param comment The column comment. + * @return This builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Build the {@link FunctionColumnDTO}. + * + * @return The function column DTO. + */ + public FunctionColumnDTO build() { + return new FunctionColumnDTO(name, dataType, comment); + } + } + + /** + * Create a new builder. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionDTO.java new file mode 100644 index 00000000000..93e4a010370 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionDTO.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.Arrays; +import javax.annotation.Nullable; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.Audit; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.rel.types.Type; + +/** Represents a Function DTO (Data Transfer Object). */ +@Getter +@EqualsAndHashCode +public class FunctionDTO implements Function { + + @JsonProperty("name") + private String name; + + @JsonProperty("functionType") + private FunctionType functionType; + + @JsonProperty("deterministic") + private boolean deterministic; + + @Nullable + @JsonProperty("comment") + private String comment; + + @Nullable + @JsonProperty("returnType") + @JsonSerialize(using = JsonUtils.TypeSerializer.class) + @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) + private Type returnType; + + @JsonProperty("returnColumns") + private FunctionColumnDTO[] returnColumns; + + @JsonProperty("definitions") + private FunctionDefinitionDTO[] definitions; + + @JsonProperty("version") + private int version; + + @JsonProperty("audit") + private AuditDTO audit; + + private FunctionDTO() {} + + private FunctionDTO( + String name, + FunctionType functionType, + boolean deterministic, + String comment, + Type returnType, + FunctionColumnDTO[] returnColumns, + FunctionDefinitionDTO[] definitions, + int version, + AuditDTO audit) { + this.name = name; + this.functionType = functionType; + this.deterministic = deterministic; + this.comment = comment; + this.returnType = returnType; + this.returnColumns = returnColumns; + this.definitions = definitions; + this.version = version; + this.audit = audit; + } + + @Override + public String name() { + return name; + } + + @Override + public FunctionType functionType() { + return functionType; + } + + @Override + public boolean deterministic() { + return deterministic; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Type returnType() { + return returnType; + } + + @Override + public FunctionColumn[] returnColumns() { + if (returnColumns == null) { + return new FunctionColumn[0]; + } + return Arrays.stream(returnColumns) + .map(FunctionColumnDTO::toFunctionColumn) + .toArray(FunctionColumn[]::new); + } + + @Override + public FunctionDefinition[] definitions() { + if (definitions == null) { + return new FunctionDefinition[0]; + } + return definitions; + } + + @Override + public int version() { + return version; + } + + @Override + public Audit auditInfo() { + return audit; + } + + @Override + public String toString() { + return "FunctionDTO{" + + "name='" + + name + + '\'' + + ", functionType=" + + functionType + + ", deterministic=" + + deterministic + + ", comment='" + + comment + + '\'' + + ", returnType=" + + returnType + + ", returnColumns=" + + Arrays.toString(returnColumns) + + ", definitions=" + + Arrays.toString(definitions) + + ", version=" + + version + + ", audit=" + + audit + + '}'; + } + + /** Builder for {@link FunctionDTO}. */ + public static class Builder { + private String name; + private FunctionType functionType; + private boolean deterministic; + private String comment; + private Type returnType; + private FunctionColumnDTO[] returnColumns; + private FunctionDefinitionDTO[] definitions; + private int version; + private AuditDTO audit; + + /** + * Set the function name. + * + * @param name The function name. + * @return This builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Set the function type. + * + * @param functionType The function type. + * @return This builder. + */ + public Builder withFunctionType(FunctionType functionType) { + this.functionType = functionType; + return this; + } + + /** + * Set whether the function is deterministic. + * + * @param deterministic Whether the function is deterministic. + * @return This builder. + */ + public Builder withDeterministic(boolean deterministic) { + this.deterministic = deterministic; + return this; + } + + /** + * Set the function comment. + * + * @param comment The function comment. + * @return This builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Set the return type. + * + * @param returnType The return type. + * @return This builder. + */ + public Builder withReturnType(Type returnType) { + this.returnType = returnType; + return this; + } + + /** + * Set the return columns for table-valued functions. + * + * @param returnColumns The return columns. + * @return This builder. + */ + public Builder withReturnColumns(FunctionColumnDTO[] returnColumns) { + this.returnColumns = returnColumns; + return this; + } + + /** + * Set the function definitions. + * + * @param definitions The function definitions. + * @return This builder. + */ + public Builder withDefinitions(FunctionDefinitionDTO[] definitions) { + this.definitions = definitions; + return this; + } + + /** + * Set the function version. + * + * @param version The function version. + * @return This builder. + */ + public Builder withVersion(int version) { + this.version = version; + return this; + } + + /** + * Set the audit information. + * + * @param audit The audit information. + * @return This builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Build the {@link FunctionDTO}. + * + * @return The function DTO. + */ + public FunctionDTO build() { + return new FunctionDTO( + name, + functionType, + deterministic, + comment, + returnType, + returnColumns, + definitions, + version, + audit); + } + } + + /** + * Create a new builder. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionDefinitionDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionDefinitionDTO.java new file mode 100644 index 00000000000..6191cdc38c0 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionDefinitionDTO.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionDefinitions; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionParam; + +/** DTO for function definition. */ +@Getter +@EqualsAndHashCode +public class FunctionDefinitionDTO implements FunctionDefinition { + + @JsonProperty("parameters") + private FunctionParamDTO[] parameters; + + @JsonProperty("impls") + private FunctionImplDTO[] impls; + + private FunctionDefinitionDTO() {} + + private FunctionDefinitionDTO(FunctionParamDTO[] parameters, FunctionImplDTO[] impls) { + this.parameters = parameters; + this.impls = impls; + } + + @Override + public FunctionParam[] parameters() { + return parameters; + } + + @Override + public FunctionImpl[] impls() { + if (impls == null) { + return new FunctionImpl[0]; + } + return Arrays.stream(impls).map(FunctionImplDTO::toFunctionImpl).toArray(FunctionImpl[]::new); + } + + /** + * Convert this DTO to a {@link FunctionDefinition} instance. + * + * @return The function definition. + */ + public FunctionDefinition toFunctionDefinition() { + FunctionParam[] params = + parameters == null + ? new FunctionParam[0] + : Arrays.stream(parameters) + .map(FunctionParamDTO::toFunctionParam) + .toArray(FunctionParam[]::new); + FunctionImpl[] implArr = + impls == null + ? new FunctionImpl[0] + : Arrays.stream(impls) + .map(FunctionImplDTO::toFunctionImpl) + .toArray(FunctionImpl[]::new); + return FunctionDefinitions.of(params, implArr); + } + + /** + * Create a {@link FunctionDefinitionDTO} from a {@link FunctionDefinition} instance. + * + * @param definition The function definition. + * @return The function definition DTO. + */ + public static FunctionDefinitionDTO fromFunctionDefinition(FunctionDefinition definition) { + FunctionParamDTO[] paramDTOs = + definition.parameters() == null + ? new FunctionParamDTO[0] + : Arrays.stream(definition.parameters()) + .map( + param -> + param instanceof FunctionParamDTO + ? (FunctionParamDTO) param + : FunctionParamDTO.fromFunctionParam(param)) + .toArray(FunctionParamDTO[]::new); + FunctionImplDTO[] implDTOs = + definition.impls() == null + ? new FunctionImplDTO[0] + : Arrays.stream(definition.impls()) + .map(FunctionImplDTO::fromFunctionImpl) + .toArray(FunctionImplDTO[]::new); + return new FunctionDefinitionDTO(paramDTOs, implDTOs); + } + + @Override + public String toString() { + return "FunctionDefinitionDTO{" + + "parameters=" + + Arrays.toString(parameters) + + ", impls=" + + Arrays.toString(impls) + + '}'; + } + + /** Builder for {@link FunctionDefinitionDTO}. */ + public static class Builder { + private FunctionParamDTO[] parameters; + private FunctionImplDTO[] impls; + + /** + * Set the parameters. + * + * @param parameters The parameters. + * @return This builder. + */ + public Builder withParameters(FunctionParamDTO[] parameters) { + this.parameters = parameters; + return this; + } + + /** + * Set the implementations. + * + * @param impls The implementations. + * @return This builder. + */ + public Builder withImpls(FunctionImplDTO[] impls) { + this.impls = impls; + return this; + } + + /** + * Build the {@link FunctionDefinitionDTO}. + * + * @return The function definition DTO. + */ + public FunctionDefinitionDTO build() { + return new FunctionDefinitionDTO(parameters, impls); + } + } + + /** + * Create a new builder. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionImplDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionImplDTO.java new file mode 100644 index 00000000000..6aef7a1417a --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionImplDTO.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.JavaImpl; +import org.apache.gravitino.function.PythonImpl; +import org.apache.gravitino.function.SQLImpl; + +/** DTO for function implementation. */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "language") +@JsonSubTypes({ + @JsonSubTypes.Type(value = SQLImplDTO.class, name = "SQL"), + @JsonSubTypes.Type(value = JavaImplDTO.class, name = "JAVA"), + @JsonSubTypes.Type(value = PythonImplDTO.class, name = "PYTHON") +}) +@Getter +@EqualsAndHashCode +public abstract class FunctionImplDTO { + + @JsonProperty("runtime") + private String runtime; + + @JsonProperty("resources") + private FunctionResourcesDTO resources; + + @JsonProperty("properties") + private Map properties; + + /** Default constructor for Jackson. */ + protected FunctionImplDTO() {} + + /** + * Constructor for FunctionImplDTO. + * + * @param runtime The runtime type. + * @param resources The function resources. + * @param properties The properties. + */ + protected FunctionImplDTO( + String runtime, FunctionResourcesDTO resources, Map properties) { + this.runtime = runtime; + this.resources = resources; + this.properties = properties; + } + + /** + * Get the language of this implementation. + * + * @return The language. + */ + public abstract FunctionImpl.Language language(); + + /** + * Convert this DTO to a {@link FunctionImpl} instance. + * + * @return The function implementation. + */ + public abstract FunctionImpl toFunctionImpl(); + + /** + * Create a {@link FunctionImplDTO} from a {@link FunctionImpl} instance. + * + * @param impl The function implementation. + * @return The function implementation DTO. + */ + public static FunctionImplDTO fromFunctionImpl(FunctionImpl impl) { + if (impl instanceof SQLImpl) { + SQLImpl sqlImpl = (SQLImpl) impl; + return new SQLImplDTO( + sqlImpl.runtime().name(), + FunctionResourcesDTO.fromFunctionResources(sqlImpl.resources()), + sqlImpl.properties(), + sqlImpl.sql()); + } else if (impl instanceof JavaImpl) { + JavaImpl javaImpl = (JavaImpl) impl; + return new JavaImplDTO( + javaImpl.runtime().name(), + FunctionResourcesDTO.fromFunctionResources(javaImpl.resources()), + javaImpl.properties(), + javaImpl.className()); + } else if (impl instanceof PythonImpl) { + PythonImpl pythonImpl = (PythonImpl) impl; + return new PythonImplDTO( + pythonImpl.runtime().name(), + FunctionResourcesDTO.fromFunctionResources(pythonImpl.resources()), + pythonImpl.properties(), + pythonImpl.handler(), + pythonImpl.codeBlock()); + } + throw new IllegalArgumentException("Unsupported implementation type: " + impl.getClass()); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionParamDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionParamDTO.java new file mode 100644 index 00000000000..ff9795c1750 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionParamDTO.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import static org.apache.gravitino.dto.util.DTOConverters.toFunctionArg; +import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import javax.annotation.Nullable; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionParam; +import org.apache.gravitino.function.FunctionParams; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.expressions.Expression; +import org.apache.gravitino.rel.types.Type; + +/** DTO for function parameter. */ +@Getter +@EqualsAndHashCode +public class FunctionParamDTO implements FunctionParam { + + @JsonProperty("name") + private String name; + + @JsonProperty("dataType") + @JsonSerialize(using = JsonUtils.TypeSerializer.class) + @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) + private Type dataType; + + @Nullable + @JsonProperty("comment") + private String comment; + + @JsonProperty("defaultValue") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonSerialize(using = JsonUtils.ColumnDefaultValueSerializer.class) + @JsonDeserialize(using = JsonUtils.ColumnDefaultValueDeserializer.class) + private Expression defaultValue = DEFAULT_VALUE_NOT_SET; + + private FunctionParamDTO() {} + + private FunctionParamDTO(String name, Type dataType, String comment, Expression defaultValue) { + this.name = name; + this.dataType = dataType; + this.comment = comment; + this.defaultValue = defaultValue == null ? DEFAULT_VALUE_NOT_SET : defaultValue; + } + + @Override + public String name() { + return name; + } + + @Override + public Type dataType() { + return dataType; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Expression defaultValue() { + return defaultValue; + } + + /** + * Convert this DTO to a {@link FunctionParam} instance. + * + * @return The function parameter. + */ + public FunctionParam toFunctionParam() { + return FunctionParams.of(name, dataType, comment, defaultValue()); + } + + /** + * Create a {@link FunctionParamDTO} from a {@link FunctionParam} instance. + * + * @param param The function parameter. + * @return The function parameter DTO. + */ + public static FunctionParamDTO fromFunctionParam(FunctionParam param) { + return new FunctionParamDTO( + param.name(), + param.dataType(), + param.comment(), + (param.defaultValue() == null || param.defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET)) + ? Column.DEFAULT_VALUE_NOT_SET + : toFunctionArg(param.defaultValue())); + } + + @Override + public String toString() { + return "FunctionParamDTO{" + + "name='" + + name + + '\'' + + ", dataType=" + + dataType + + ", comment='" + + comment + + '\'' + + ", defaultValue=" + + defaultValue + + '}'; + } + + /** Builder for {@link FunctionParamDTO}. */ + public static class Builder { + private String name; + private Type dataType; + private String comment; + private Expression defaultValue; + + /** + * Set the parameter name. + * + * @param name The parameter name. + * @return This builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Set the parameter data type. + * + * @param dataType The parameter data type. + * @return This builder. + */ + public Builder withDataType(Type dataType) { + this.dataType = dataType; + return this; + } + + /** + * Set the parameter comment. + * + * @param comment The parameter comment. + * @return This builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Set the parameter default value. + * + * @param defaultValue The parameter default value. + * @return This builder. + */ + public Builder withDefaultValue(Expression defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + /** + * Build the {@link FunctionParamDTO}. + * + * @return The function parameter DTO. + */ + public FunctionParamDTO build() { + return new FunctionParamDTO(name, dataType, comment, defaultValue); + } + } + + /** + * Create a new builder. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/FunctionResourcesDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/FunctionResourcesDTO.java new file mode 100644 index 00000000000..fce8ba93787 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/FunctionResourcesDTO.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionResources; + +/** DTO for function resources. */ +@Getter +@EqualsAndHashCode +public class FunctionResourcesDTO { + + @JsonProperty("jars") + private String[] jars; + + @JsonProperty("files") + private String[] files; + + @JsonProperty("archives") + private String[] archives; + + private FunctionResourcesDTO() {} + + private FunctionResourcesDTO(String[] jars, String[] files, String[] archives) { + this.jars = jars; + this.files = files; + this.archives = archives; + } + + /** + * Convert this DTO to a {@link FunctionResources} instance. + * + * @return The function resources. + */ + public FunctionResources toFunctionResources() { + return FunctionResources.of(jars, files, archives); + } + + /** + * Create a {@link FunctionResourcesDTO} from a {@link FunctionResources} instance. + * + * @param resources The function resources. + * @return The function resources DTO. + */ + public static FunctionResourcesDTO fromFunctionResources(FunctionResources resources) { + if (resources == null) { + return null; + } + return new FunctionResourcesDTO(resources.jars(), resources.files(), resources.archives()); + } + + @Override + public String toString() { + return "FunctionResourcesDTO{" + + "jars=" + + Arrays.toString(jars) + + ", files=" + + Arrays.toString(files) + + ", archives=" + + Arrays.toString(archives) + + '}'; + } + + /** Builder for {@link FunctionResourcesDTO}. */ + public static class Builder { + private String[] jars; + private String[] files; + private String[] archives; + + /** + * Set the jar resources. + * + * @param jars The jar resources. + * @return This builder. + */ + public Builder withJars(String[] jars) { + this.jars = jars; + return this; + } + + /** + * Set the file resources. + * + * @param files The file resources. + * @return This builder. + */ + public Builder withFiles(String[] files) { + this.files = files; + return this; + } + + /** + * Set the archive resources. + * + * @param archives The archive resources. + * @return This builder. + */ + public Builder withArchives(String[] archives) { + this.archives = archives; + return this; + } + + /** + * Build the {@link FunctionResourcesDTO}. + * + * @return The function resources DTO. + */ + public FunctionResourcesDTO build() { + return new FunctionResourcesDTO(jars, files, archives); + } + } + + /** + * Create a new builder. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/JavaImplDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/JavaImplDTO.java new file mode 100644 index 00000000000..33e0d02ab58 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/JavaImplDTO.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; + +/** Java implementation DTO. */ +@Getter +@EqualsAndHashCode(callSuper = true) +public class JavaImplDTO extends FunctionImplDTO { + + @JsonProperty("className") + private String className; + + private JavaImplDTO() {} + + /** + * Constructor for JavaImplDTO. + * + * @param runtime The runtime type. + * @param resources The function resources. + * @param properties The properties. + * @param className The fully qualified class name. + */ + public JavaImplDTO( + String runtime, + FunctionResourcesDTO resources, + Map properties, + String className) { + super(runtime, resources, properties); + this.className = className; + } + + @Override + public FunctionImpl.Language language() { + return FunctionImpl.Language.JAVA; + } + + @Override + public FunctionImpl toFunctionImpl() { + return FunctionImpls.ofJava( + FunctionImpl.RuntimeType.fromString(getRuntime()), + className, + getResources() != null ? getResources().toFunctionResources() : null, + getProperties()); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/PythonImplDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/PythonImplDTO.java new file mode 100644 index 00000000000..e9b38616be7 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/PythonImplDTO.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; + +/** Python implementation DTO. */ +@Getter +@EqualsAndHashCode(callSuper = true) +public class PythonImplDTO extends FunctionImplDTO { + + @JsonProperty("handler") + private String handler; + + @JsonProperty("codeBlock") + private String codeBlock; + + private PythonImplDTO() {} + + /** + * Constructor for PythonImplDTO. + * + * @param runtime The runtime type. + * @param resources The function resources. + * @param properties The properties. + * @param handler The Python handler function. + * @param codeBlock The Python code block. + */ + public PythonImplDTO( + String runtime, + FunctionResourcesDTO resources, + Map properties, + String handler, + String codeBlock) { + super(runtime, resources, properties); + this.handler = handler; + this.codeBlock = codeBlock; + } + + @Override + public FunctionImpl.Language language() { + return FunctionImpl.Language.PYTHON; + } + + @Override + public FunctionImpl toFunctionImpl() { + return FunctionImpls.ofPython( + FunctionImpl.RuntimeType.fromString(getRuntime()), + handler, + codeBlock, + getResources() != null ? getResources().toFunctionResources() : null, + getProperties()); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/function/SQLImplDTO.java b/common/src/main/java/org/apache/gravitino/dto/function/SQLImplDTO.java new file mode 100644 index 00000000000..87f69a83f93 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/function/SQLImplDTO.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; + +/** SQL implementation DTO. */ +@Getter +@EqualsAndHashCode(callSuper = true) +public class SQLImplDTO extends FunctionImplDTO { + + @JsonProperty("sql") + private String sql; + + private SQLImplDTO() { + super(); + } + + /** + * Constructor for SQLImplDTO. + * + * @param runtime The runtime type. + * @param resources The function resources. + * @param properties The properties. + * @param sql The SQL expression. + */ + public SQLImplDTO( + String runtime, FunctionResourcesDTO resources, Map properties, String sql) { + super(runtime, resources, properties); + this.sql = sql; + } + + @Override + public FunctionImpl.Language language() { + return FunctionImpl.Language.SQL; + } + + @Override + public FunctionImpl toFunctionImpl() { + return FunctionImpls.ofSql( + FunctionImpl.RuntimeType.fromString(getRuntime()), + sql, + getResources() != null ? getResources().toFunctionResources() : null, + getProperties()); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/FunctionRegisterRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionRegisterRequest.java new file mode 100644 index 00000000000..4df647a7db5 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionRegisterRequest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.Preconditions; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.function.FunctionColumnDTO; +import org.apache.gravitino.dto.function.FunctionDefinitionDTO; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to register a function. */ +@Getter +@EqualsAndHashCode +@ToString +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FunctionRegisterRequest implements RESTRequest { + + @JsonProperty("name") + private String name; + + @JsonProperty("functionType") + private FunctionType functionType; + + @JsonProperty("deterministic") + private boolean deterministic; + + @Nullable + @JsonProperty("comment") + private String comment; + + @Nullable + @JsonProperty("returnType") + @JsonSerialize(using = JsonUtils.TypeSerializer.class) + @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) + private Type returnType; + + @Nullable + @JsonProperty("returnColumns") + private FunctionColumnDTO[] returnColumns; + + @JsonProperty("definitions") + private FunctionDefinitionDTO[] definitions; + + /** + * Validates the request. + * + * @throws IllegalArgumentException if the request is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" field is required and cannot be empty"); + Preconditions.checkArgument(functionType != null, "\"functionType\" field is required"); + Preconditions.checkArgument( + definitions != null && definitions.length > 0, + "\"definitions\" field is required and cannot be empty"); + + if (functionType == FunctionType.TABLE) { + Preconditions.checkArgument( + returnColumns != null && returnColumns.length > 0, + "\"returnColumns\" is required for TABLE function type"); + } else { + Preconditions.checkArgument( + returnType != null, "\"returnType\" is required for SCALAR or AGGREGATE function type"); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdateRequest.java new file mode 100644 index 00000000000..76f368f573c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdateRequest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Preconditions; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.gravitino.dto.function.FunctionDefinitionDTO; +import org.apache.gravitino.dto.function.FunctionImplDTO; +import org.apache.gravitino.dto.function.FunctionParamDTO; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionParam; +import org.apache.gravitino.rest.RESTRequest; + +/** Request to update a function. */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) +@JsonSubTypes({ + @JsonSubTypes.Type( + value = FunctionUpdateRequest.UpdateCommentRequest.class, + name = "updateComment"), + @JsonSubTypes.Type( + value = FunctionUpdateRequest.AddDefinitionRequest.class, + name = "addDefinition"), + @JsonSubTypes.Type( + value = FunctionUpdateRequest.RemoveDefinitionRequest.class, + name = "removeDefinition"), + @JsonSubTypes.Type(value = FunctionUpdateRequest.AddImplRequest.class, name = "addImpl"), + @JsonSubTypes.Type(value = FunctionUpdateRequest.UpdateImplRequest.class, name = "updateImpl"), + @JsonSubTypes.Type(value = FunctionUpdateRequest.RemoveImplRequest.class, name = "removeImpl") +}) +public interface FunctionUpdateRequest extends RESTRequest { + + /** + * Returns the function change. + * + * @return the function change. + */ + FunctionChange functionChange(); + + /** The function update request for updating the comment of a function. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class UpdateCommentRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("newComment") + private final String newComment; + + @Override + public FunctionChange functionChange() { + return FunctionChange.updateComment(newComment); + } + + @Override + public void validate() throws IllegalArgumentException {} + } + + /** The function update request for adding a definition to a function. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class AddDefinitionRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("definition") + private final FunctionDefinitionDTO definition; + + @Override + public FunctionChange functionChange() { + FunctionDefinition def = definition.toFunctionDefinition(); + return FunctionChange.addDefinition(def); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + definition != null, "\"definition\" field is required and cannot be null"); + } + } + + /** The function update request for removing a definition from a function. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class RemoveDefinitionRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("parameters") + private final FunctionParamDTO[] parameters; + + @Override + public FunctionChange functionChange() { + FunctionParam[] params = new FunctionParam[parameters == null ? 0 : parameters.length]; + if (parameters != null) { + for (int i = 0; i < parameters.length; i++) { + params[i] = parameters[i].toFunctionParam(); + } + } + return FunctionChange.removeDefinition(params); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + parameters != null, "\"parameters\" field is required and cannot be null"); + } + } + + /** The function update request for adding an implementation to a definition. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class AddImplRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("parameters") + private final FunctionParamDTO[] parameters; + + @Getter + @JsonProperty("implementation") + private final FunctionImplDTO implementation; + + @Override + public FunctionChange functionChange() { + FunctionParam[] params = new FunctionParam[parameters == null ? 0 : parameters.length]; + if (parameters != null) { + for (int i = 0; i < parameters.length; i++) { + params[i] = parameters[i].toFunctionParam(); + } + } + FunctionImpl impl = implementation.toFunctionImpl(); + return FunctionChange.addImpl(params, impl); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + parameters != null, "\"parameters\" field is required and cannot be null"); + Preconditions.checkArgument( + implementation != null, "\"implementation\" field is required and cannot be null"); + } + } + + /** The function update request for updating an implementation in a definition. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class UpdateImplRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("parameters") + private final FunctionParamDTO[] parameters; + + @Getter + @JsonProperty("runtime") + private final String runtime; + + @Getter + @JsonProperty("implementation") + private final FunctionImplDTO implementation; + + @Override + public FunctionChange functionChange() { + FunctionParam[] params = new FunctionParam[parameters == null ? 0 : parameters.length]; + if (parameters != null) { + for (int i = 0; i < parameters.length; i++) { + params[i] = parameters[i].toFunctionParam(); + } + } + FunctionImpl.RuntimeType runtimeType = FunctionImpl.RuntimeType.fromString(runtime); + FunctionImpl impl = implementation.toFunctionImpl(); + return FunctionChange.updateImpl(params, runtimeType, impl); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + parameters != null, "\"parameters\" field is required and cannot be null"); + Preconditions.checkArgument( + runtime != null, "\"runtime\" field is required and cannot be null"); + Preconditions.checkArgument( + implementation != null, "\"implementation\" field is required and cannot be null"); + } + } + + /** The function update request for removing an implementation from a definition. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @AllArgsConstructor + @ToString + class RemoveImplRequest implements FunctionUpdateRequest { + + @Getter + @JsonProperty("parameters") + private final FunctionParamDTO[] parameters; + + @Getter + @JsonProperty("runtime") + private final String runtime; + + @Override + public FunctionChange functionChange() { + FunctionParam[] params = new FunctionParam[parameters == null ? 0 : parameters.length]; + if (parameters != null) { + for (int i = 0; i < parameters.length; i++) { + params[i] = parameters[i].toFunctionParam(); + } + } + FunctionImpl.RuntimeType runtimeType = FunctionImpl.RuntimeType.fromString(runtime); + return FunctionChange.removeImpl(params, runtimeType); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + parameters != null, "\"parameters\" field is required and cannot be null"); + Preconditions.checkArgument( + runtime != null, "\"runtime\" field is required and cannot be null"); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdatesRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdatesRequest.java new file mode 100644 index 00000000000..2f5468234fe --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/FunctionUpdatesRequest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.gravitino.rest.RESTMessage; +import org.apache.gravitino.rest.RESTRequest; + +/** Request to represent updates to a function. */ +@Getter +@EqualsAndHashCode +@NoArgsConstructor(force = true) +@AllArgsConstructor +@ToString +public class FunctionUpdatesRequest implements RESTRequest { + + @JsonProperty("updates") + private final List updates; + + @Override + public void validate() throws IllegalArgumentException { + if (updates == null) { + throw new IllegalArgumentException("Updates list cannot be null"); + } + updates.forEach(RESTMessage::validate); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/FunctionListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/FunctionListResponse.java new file mode 100644 index 00000000000..d9cbe2d727e --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/FunctionListResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.function.FunctionDTO; + +/** Response wrapper for multiple functions. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class FunctionListResponse extends BaseResponse { + + @JsonProperty("functions") + private FunctionDTO[] functions; + + /** + * Creates a response containing multiple functions. + * + * @param functions Function array payload. + */ + public FunctionListResponse(FunctionDTO[] functions) { + super(0); + this.functions = functions; + } + + private FunctionListResponse() { + super(); + } + + /** {@inheritDoc} */ + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(functions != null, "functions must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/FunctionResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/FunctionResponse.java new file mode 100644 index 00000000000..0566aac300f --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/FunctionResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.function.FunctionDTO; + +/** Response for function operations. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class FunctionResponse extends BaseResponse { + + @JsonProperty("function") + private final FunctionDTO function; + + /** Constructor for FunctionResponse. */ + public FunctionResponse() { + super(0); + this.function = null; + } + + /** + * Constructor for FunctionResponse. + * + * @param function the function DTO object. + */ + public FunctionResponse(FunctionDTO function) { + super(0); + this.function = function; + } + + /** + * Validates the response. + * + * @throws IllegalArgumentException if the response is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(function != null, "function must not be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(function.name()), "function 'name' must not be null and empty"); + Preconditions.checkArgument( + function.functionType() != null, "function 'functionType' must not be null"); + Preconditions.checkArgument( + function.definitions() != null, "function 'definitions' must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index 29ac24fec79..165a7aa02f1 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -51,6 +51,9 @@ import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.file.FileInfoDTO; import org.apache.gravitino.dto.file.FilesetDTO; +import org.apache.gravitino.dto.function.FunctionColumnDTO; +import org.apache.gravitino.dto.function.FunctionDTO; +import org.apache.gravitino.dto.function.FunctionDefinitionDTO; import org.apache.gravitino.dto.job.JobTemplateDTO; import org.apache.gravitino.dto.job.ShellJobTemplateDTO; import org.apache.gravitino.dto.job.SparkJobTemplateDTO; @@ -88,6 +91,7 @@ import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.FileInfo; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.function.Function; import org.apache.gravitino.job.JobTemplate; import org.apache.gravitino.job.ShellJobTemplate; import org.apache.gravitino.job.SparkJobTemplate; @@ -726,6 +730,42 @@ public static ModelVersionDTO toDTO(ModelVersion modelVersion) { .build(); } + /** + * Converts a Function to a FunctionDTO. + * + * @param function The function to be converted. + * @return The function DTO. + */ + public static FunctionDTO toDTO(Function function) { + FunctionColumnDTO[] returnColumnDTOs = null; + if (function.returnColumns() != null && function.returnColumns().length > 0) { + returnColumnDTOs = + Arrays.stream(function.returnColumns()) + .map(FunctionColumnDTO::fromFunctionColumn) + .toArray(FunctionColumnDTO[]::new); + } + + FunctionDefinitionDTO[] definitionDTOs = null; + if (function.definitions() != null) { + definitionDTOs = + Arrays.stream(function.definitions()) + .map(FunctionDefinitionDTO::fromFunctionDefinition) + .toArray(FunctionDefinitionDTO[]::new); + } + + return FunctionDTO.builder() + .withName(function.name()) + .withFunctionType(function.functionType()) + .withDeterministic(function.deterministic()) + .withComment(function.comment()) + .withReturnType(function.returnType()) + .withReturnColumns(returnColumnDTOs) + .withDefinitions(definitionDTOs) + .withVersion(function.version()) + .withAudit(toDTO(function.auditInfo())) + .build(); + } + /** * Converts an array of ModelVersions to an array of ModelVersionDTOs. * diff --git a/common/src/test/java/org/apache/gravitino/dto/function/TestFunctionDTO.java b/common/src/test/java/org/apache/gravitino/dto/function/TestFunctionDTO.java new file mode 100644 index 00000000000..0a1c29cf496 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/function/TestFunctionDTO.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.function; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.dto.rel.expressions.LiteralDTO; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.rel.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestFunctionDTO { + + @Test + public void testFunctionDTOSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + // Create function parameter with default value + FunctionParamDTO param1 = + FunctionParamDTO.builder() + .withName("x") + .withDataType(Types.IntegerType.get()) + .withComment("input parameter") + .build(); + + FunctionParamDTO param2 = + FunctionParamDTO.builder() + .withName("y") + .withDataType(Types.FloatType.get()) + .withDefaultValue( + LiteralDTO.builder().withDataType(Types.FloatType.get()).withValue("1.0").build()) + .build(); + + // Create SQL implementation + SQLImplDTO sqlImpl = + new SQLImplDTO( + FunctionImpl.RuntimeType.SPARK.name(), + null, + ImmutableMap.of("key", "value"), + "SELECT x + y"); + + // Create function definition + FunctionDefinitionDTO definition = + FunctionDefinitionDTO.builder() + .withParameters(new FunctionParamDTO[] {param1, param2}) + .withImpls(new FunctionImplDTO[] {sqlImpl}) + .build(); + + // Create scalar function DTO + FunctionDTO scalarFunction = + FunctionDTO.builder() + .withName("add_func") + .withFunctionType(FunctionType.SCALAR) + .withDeterministic(true) + .withComment("A simple add function") + .withReturnType(Types.FloatType.get()) + .withDefinitions(new FunctionDefinitionDTO[] {definition}) + .withVersion(1) + .withAudit(audit) + .build(); + + // Serialize and deserialize + String json = JsonUtils.objectMapper().writeValueAsString(scalarFunction); + FunctionDTO deserialized = JsonUtils.objectMapper().readValue(json, FunctionDTO.class); + + Assertions.assertEquals(scalarFunction.name(), deserialized.name()); + Assertions.assertEquals(scalarFunction.functionType(), deserialized.functionType()); + Assertions.assertEquals(scalarFunction.deterministic(), deserialized.deterministic()); + Assertions.assertEquals(scalarFunction.comment(), deserialized.comment()); + Assertions.assertEquals(scalarFunction.returnType(), deserialized.returnType()); + Assertions.assertEquals(scalarFunction.version(), deserialized.version()); + Assertions.assertEquals(scalarFunction.definitions().length, deserialized.definitions().length); + } + + @Test + public void testTableFunctionDTOSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + // Create return columns for table function + FunctionColumnDTO col1 = + FunctionColumnDTO.builder() + .withName("id") + .withDataType(Types.IntegerType.get()) + .withComment("id column") + .build(); + + FunctionColumnDTO col2 = + FunctionColumnDTO.builder().withName("name").withDataType(Types.StringType.get()).build(); + + // Create Java implementation + JavaImplDTO javaImpl = + new JavaImplDTO( + FunctionImpl.RuntimeType.SPARK.name(), + FunctionResourcesDTO.builder() + .withJars(new String[] {"hdfs://path/to/jar.jar"}) + .build(), + ImmutableMap.of(), + "com.example.TableFunction"); + + // Create function definition + FunctionDefinitionDTO definition = + FunctionDefinitionDTO.builder() + .withParameters(new FunctionParamDTO[] {}) + .withImpls(new FunctionImplDTO[] {javaImpl}) + .build(); + + // Create table function DTO + FunctionDTO tableFunction = + FunctionDTO.builder() + .withName("table_func") + .withFunctionType(FunctionType.TABLE) + .withDeterministic(false) + .withComment("A table function") + .withReturnColumns(new FunctionColumnDTO[] {col1, col2}) + .withDefinitions(new FunctionDefinitionDTO[] {definition}) + .withVersion(1) + .withAudit(audit) + .build(); + + // Serialize and deserialize + String json = JsonUtils.objectMapper().writeValueAsString(tableFunction); + FunctionDTO deserialized = JsonUtils.objectMapper().readValue(json, FunctionDTO.class); + + Assertions.assertEquals(tableFunction.name(), deserialized.name()); + Assertions.assertEquals(FunctionType.TABLE, deserialized.functionType()); + Assertions.assertEquals(2, deserialized.returnColumns().length); + } + + @Test + public void testFunctionImplDTOSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + FunctionResourcesDTO resources = + FunctionResourcesDTO.builder() + .withJars(new String[] {"hdfs://path/to/jar.jar"}) + .withFiles(new String[] {"hdfs://path/to/file.txt"}) + .build(); + + // Test SQL implementation + SQLImplDTO sqlImpl = new SQLImplDTO("SPARK", resources, props, "SELECT 1"); + String sqlJson = JsonUtils.objectMapper().writeValueAsString(sqlImpl); + FunctionImplDTO deserializedSql = + JsonUtils.objectMapper().readValue(sqlJson, FunctionImplDTO.class); + Assertions.assertTrue(deserializedSql instanceof SQLImplDTO); + Assertions.assertEquals(FunctionImpl.Language.SQL, deserializedSql.language()); + Assertions.assertEquals("SELECT 1", ((SQLImplDTO) deserializedSql).getSql()); + + // Test Java implementation + JavaImplDTO javaImpl = new JavaImplDTO("SPARK", resources, props, "com.example.MyUDF"); + String javaJson = JsonUtils.objectMapper().writeValueAsString(javaImpl); + FunctionImplDTO deserializedJava = + JsonUtils.objectMapper().readValue(javaJson, FunctionImplDTO.class); + Assertions.assertTrue(deserializedJava instanceof JavaImplDTO); + Assertions.assertEquals(FunctionImpl.Language.JAVA, deserializedJava.language()); + Assertions.assertEquals("com.example.MyUDF", ((JavaImplDTO) deserializedJava).getClassName()); + + // Test Python implementation + PythonImplDTO pythonImpl = + new PythonImplDTO( + "SPARK", resources, props, "my_handler", "def my_handler(x): return x * 2"); + String pythonJson = JsonUtils.objectMapper().writeValueAsString(pythonImpl); + FunctionImplDTO deserializedPython = + JsonUtils.objectMapper().readValue(pythonJson, FunctionImplDTO.class); + Assertions.assertTrue(deserializedPython instanceof PythonImplDTO); + Assertions.assertEquals(FunctionImpl.Language.PYTHON, deserializedPython.language()); + Assertions.assertEquals("my_handler", ((PythonImplDTO) deserializedPython).getHandler()); + } + + @Test + public void testFunctionParamDTOWithDefaultValue() throws JsonProcessingException { + // Test parameter without default value + FunctionParamDTO paramWithoutDefault = + FunctionParamDTO.builder() + .withName("x") + .withDataType(Types.IntegerType.get()) + .withComment("no default") + .build(); + + String json1 = JsonUtils.objectMapper().writeValueAsString(paramWithoutDefault); + FunctionParamDTO deserialized1 = + JsonUtils.objectMapper().readValue(json1, FunctionParamDTO.class); + Assertions.assertEquals("x", deserialized1.name()); + Assertions.assertEquals(Types.IntegerType.get(), deserialized1.dataType()); + + // Test parameter with default value + LiteralDTO defaultValue = + LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("10").build(); + + FunctionParamDTO paramWithDefault = + FunctionParamDTO.builder() + .withName("y") + .withDataType(Types.IntegerType.get()) + .withDefaultValue(defaultValue) + .build(); + + String json2 = JsonUtils.objectMapper().writeValueAsString(paramWithDefault); + FunctionParamDTO deserialized2 = + JsonUtils.objectMapper().readValue(json2, FunctionParamDTO.class); + Assertions.assertEquals("y", deserialized2.name()); + Assertions.assertNotNull(deserialized2.defaultValue()); + } + + @Test + public void testFunctionResourcesDTOSerDe() throws JsonProcessingException { + FunctionResourcesDTO resources = + FunctionResourcesDTO.builder() + .withJars(new String[] {"hdfs://path/to/jar1.jar", "hdfs://path/to/jar2.jar"}) + .withFiles(new String[] {"hdfs://path/to/file.txt"}) + .withArchives(new String[] {"hdfs://path/to/archive.zip"}) + .build(); + + String json = JsonUtils.objectMapper().writeValueAsString(resources); + FunctionResourcesDTO deserialized = + JsonUtils.objectMapper().readValue(json, FunctionResourcesDTO.class); + + Assertions.assertArrayEquals(resources.getJars(), deserialized.getJars()); + Assertions.assertArrayEquals(resources.getFiles(), deserialized.getFiles()); + Assertions.assertArrayEquals(resources.getArchives(), deserialized.getArchives()); + } + + @Test + public void testFunctionDefinitionDTOSerDe() throws JsonProcessingException { + FunctionParamDTO param = + FunctionParamDTO.builder().withName("input").withDataType(Types.StringType.get()).build(); + + SQLImplDTO impl = new SQLImplDTO("SPARK", null, null, "SELECT UPPER(input)"); + + FunctionDefinitionDTO definition = + FunctionDefinitionDTO.builder() + .withParameters(new FunctionParamDTO[] {param}) + .withImpls(new FunctionImplDTO[] {impl}) + .build(); + + String json = JsonUtils.objectMapper().writeValueAsString(definition); + FunctionDefinitionDTO deserialized = + JsonUtils.objectMapper().readValue(json, FunctionDefinitionDTO.class); + + Assertions.assertEquals(1, deserialized.parameters().length); + Assertions.assertEquals("input", deserialized.parameters()[0].name()); + Assertions.assertEquals(1, deserialized.impls().length); + } +} diff --git a/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java b/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java index f79e948f227..d3e1b331f2d 100644 --- a/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java +++ b/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java @@ -212,6 +212,21 @@ public static NameIdentifier ofModel( return NameIdentifier.of(metalake, catalog, schema, model); } + /** + * Create the function {@link NameIdentifier} with the given metalake, catalog, schema and + * function name. + * + * @param metalake The metalake name + * @param catalog The catalog name + * @param schema The schema name + * @param function The function name + * @return The created function {@link NameIdentifier} + */ + public static NameIdentifier ofFunction( + String metalake, String catalog, String schema, String function) { + return NameIdentifier.of(metalake, catalog, schema, function); + } + /** * Create the model {@link NameIdentifier} from the give model version's namespace. * diff --git a/core/src/main/java/org/apache/gravitino/utils/NamespaceUtil.java b/core/src/main/java/org/apache/gravitino/utils/NamespaceUtil.java index 1578c264fad..49ea4618536 100644 --- a/core/src/main/java/org/apache/gravitino/utils/NamespaceUtil.java +++ b/core/src/main/java/org/apache/gravitino/utils/NamespaceUtil.java @@ -171,6 +171,18 @@ public static Namespace ofModel(String metalake, String catalog, String schema) return Namespace.of(metalake, catalog, schema); } + /** + * Create a namespace for function. + * + * @param metalake The metalake name + * @param catalog The catalog name + * @param schema The schema name + * @return A namespace for function + */ + public static Namespace ofFunction(String metalake, String catalog, String schema) { + return Namespace.of(metalake, catalog, schema); + } + /** * Create a namespace for model version. * diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index f86e35e2e4e..90a537f885a 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -28,6 +28,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.FunctionDispatcher; import org.apache.gravitino.catalog.ModelDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; @@ -146,6 +147,7 @@ protected void configure() { .to(CredentialOperationDispatcher.class) .ranked(1); bind(gravitinoEnv.modelDispatcher()).to(ModelDispatcher.class).ranked(1); + bind(gravitinoEnv.functionDispatcher()).to(FunctionDispatcher.class).ranked(1); bind(lineageService).to(LineageDispatcher.class).ranked(1); bind(gravitinoEnv.jobOperationDispatcher()).to(JobOperationDispatcher.class).ranked(1); bind(gravitinoEnv.statisticDispatcher()).to(StatisticDispatcher.class).ranked(1); diff --git a/server/src/main/java/org/apache/gravitino/server/web/filter/GravitinoInterceptionService.java b/server/src/main/java/org/apache/gravitino/server/web/filter/GravitinoInterceptionService.java index e974c194923..7664b7df5dd 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/filter/GravitinoInterceptionService.java +++ b/server/src/main/java/org/apache/gravitino/server/web/filter/GravitinoInterceptionService.java @@ -51,6 +51,7 @@ import org.apache.gravitino.server.web.filter.authorization.AuthorizeExecutorFactory; import org.apache.gravitino.server.web.rest.CatalogOperations; import org.apache.gravitino.server.web.rest.FilesetOperations; +import org.apache.gravitino.server.web.rest.FunctionOperations; import org.apache.gravitino.server.web.rest.GroupOperations; import org.apache.gravitino.server.web.rest.JobOperations; import org.apache.gravitino.server.web.rest.MetadataObjectCredentialOperations; @@ -92,6 +93,7 @@ public Filter getDescriptorFilter() { SchemaOperations.class.getName(), TableOperations.class.getName(), ModelOperations.class.getName(), + FunctionOperations.class.getName(), TopicOperations.class.getName(), FilesetOperations.class.getName(), UserOperations.class.getName(), diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index d3ddc1087fd..a1635798bd3 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -141,6 +141,11 @@ public static Response handleModelException( return ModelExceptionHandler.INSTANCE.handle(op, model, schema, e); } + public static Response handleFunctionException( + OperationType op, String function, String schema, Exception e) { + return FunctionExceptionHandler.INSTANCE.handle(op, function, schema, e); + } + public static Response handleJobTemplateException( OperationType op, String jobTemplate, String metalake, Exception e) { return JobTemplateExceptionHandler.INSTANCE.handle(op, jobTemplate, metalake, e); @@ -842,6 +847,43 @@ public Response handle(OperationType op, String model, String schema, Exception } } + private static class FunctionExceptionHandler extends BaseExceptionHandler { + private static final ExceptionHandler INSTANCE = new FunctionExceptionHandler(); + + private static String getFunctionErrorMsg( + String function, String operation, String schema, String reason) { + return String.format( + "Failed to operate function(s)%s operation [%s] under schema [%s], reason [%s]", + function, operation, schema, reason); + } + + @Override + public Response handle(OperationType op, String function, String schema, Exception e) { + String formatted = StringUtil.isBlank(function) ? "" : " [" + function + "]"; + String errorMsg = getFunctionErrorMsg(formatted, op.name(), schema, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof org.apache.gravitino.exceptions.FunctionAlreadyExistsException) { + return Utils.alreadyExists(errorMsg, e); + + } else if (e instanceof ForbiddenException) { + return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else { + return super.handle(op, function, schema, e); + } + } + } + private static class JobTemplateExceptionHandler extends BaseExceptionHandler { private static final ExceptionHandler INSTANCE = new JobTemplateExceptionHandler(); diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FunctionOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FunctionOperations.java new file mode 100644 index 00000000000..d072b429661 --- /dev/null +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FunctionOperations.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import com.google.common.base.Preconditions; +import java.util.Arrays; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.FunctionDispatcher; +import org.apache.gravitino.dto.function.FunctionColumnDTO; +import org.apache.gravitino.dto.function.FunctionDTO; +import org.apache.gravitino.dto.function.FunctionDefinitionDTO; +import org.apache.gravitino.dto.requests.FunctionRegisterRequest; +import org.apache.gravitino.dto.requests.FunctionUpdateRequest; +import org.apache.gravitino.dto.requests.FunctionUpdatesRequest; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.FunctionListResponse; +import org.apache.gravitino.dto.responses.FunctionResponse; +import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.server.web.Utils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** REST operations for function management. */ +@Path("metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/functions") +public class FunctionOperations { + + private static final Logger LOG = LoggerFactory.getLogger(FunctionOperations.class); + + private final FunctionDispatcher dispatcher; + + @Context private HttpServletRequest httpRequest; + + @Inject + public FunctionOperations(FunctionDispatcher dispatcher) { + this.dispatcher = dispatcher; + } + + // TODO: Add authorization support for function operations + @GET + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-function." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-function", absolute = true) + public Response listFunctions( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @QueryParam("details") @DefaultValue("false") boolean details) { + try { + LOG.info("Received list functions request for schema: {}.{}.{}", metalake, catalog, schema); + return Utils.doAs( + httpRequest, + () -> { + Namespace namespace = NamespaceUtil.ofFunction(metalake, catalog, schema); + if (!details) { + NameIdentifier[] identifiers = dispatcher.listFunctions(namespace); + LOG.info( + "List {} function names under schema: {}.{}.{}", + identifiers.length, + metalake, + catalog, + schema); + return Utils.ok(new EntityListResponse(identifiers)); + } + + Function[] functions = dispatcher.listFunctionInfos(namespace); + FunctionDTO[] functionDTOs = + Arrays.stream(functions) + .map(DTOConverters::toDTO) + .toList() + .toArray(new FunctionDTO[0]); + LOG.info( + "List {} function definitions under schema: {}.{}.{}", + functionDTOs.length, + metalake, + catalog, + schema); + return Utils.ok(new FunctionListResponse(functionDTOs)); + }); + } catch (Exception e) { + return ExceptionHandlers.handleFunctionException(OperationType.LIST, "", schema, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "register-function." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "register-function", absolute = true) + public Response registerFunction( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + FunctionRegisterRequest request) { + LOG.info( + "Received register function request: {}.{}.{}.{}", + metalake, + catalog, + schema, + request.getName()); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = + NameIdentifierUtil.ofFunction(metalake, catalog, schema, request.getName()); + + FunctionDefinition[] definitions = + Arrays.stream(request.getDefinitions()) + .map(FunctionDefinitionDTO::toFunctionDefinition) + .toArray(FunctionDefinition[]::new); + + Function function; + if (request.getFunctionType() == FunctionType.TABLE) { + FunctionColumn[] returnColumns = + Arrays.stream(request.getReturnColumns()) + .map(FunctionColumnDTO::toFunctionColumn) + .toArray(FunctionColumn[]::new); + function = + dispatcher.registerFunction( + ident, + request.getComment(), + request.isDeterministic(), + returnColumns, + definitions); + } else { + Type returnType = request.getReturnType(); + function = + dispatcher.registerFunction( + ident, + request.getComment(), + request.getFunctionType(), + request.isDeterministic(), + returnType, + definitions); + } + + Response response = Utils.ok(new FunctionResponse(DTOConverters.toDTO(function))); + LOG.info( + "Function registered: {}.{}.{}.{}", metalake, catalog, schema, request.getName()); + return response; + }); + } catch (Exception e) { + return ExceptionHandlers.handleFunctionException( + OperationType.CREATE, request.getName(), schema, e); + } + } + + @GET + @Path("{function}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-function." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-function", absolute = true) + public Response getFunction( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("function") String function, + @QueryParam("version") Integer version) { + LOG.info( + "Received get function request: {}.{}.{}.{}, version: {}", + metalake, + catalog, + schema, + function, + version); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = + NameIdentifierUtil.ofFunction(metalake, catalog, schema, function); + Function f; + if (version == null) { + f = dispatcher.getFunction(ident); + } else { + Preconditions.checkArgument( + version >= 0, "Version must be non-negative, but got:" + version); + f = dispatcher.getFunction(ident, version); + } + Response response = Utils.ok(new FunctionResponse(DTOConverters.toDTO(f))); + LOG.info("Function loaded: {}.{}.{}.{}", metalake, catalog, schema, function); + return response; + }); + } catch (Exception e) { + return ExceptionHandlers.handleFunctionException(OperationType.LOAD, function, schema, e); + } + } + + @PUT + @Path("{function}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "alter-function." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "alter-function", absolute = true) + public Response alterFunction( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("function") String function, + FunctionUpdatesRequest request) { + LOG.info("Received alter function request: {}.{}.{}.{}", metalake, catalog, schema, function); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = + NameIdentifierUtil.ofFunction(metalake, catalog, schema, function); + FunctionChange[] changes = + request.getUpdates().stream() + .map(FunctionUpdateRequest::functionChange) + .toArray(FunctionChange[]::new); + Function f = dispatcher.alterFunction(ident, changes); + Response response = Utils.ok(new FunctionResponse(DTOConverters.toDTO(f))); + LOG.info("Function altered: {}.{}.{}.{}", metalake, catalog, schema, f.name()); + return response; + }); + } catch (Exception e) { + return ExceptionHandlers.handleFunctionException(OperationType.ALTER, function, schema, e); + } + } + + @DELETE + @Path("{function}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "drop-function." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "drop-function", absolute = true) + public Response dropFunction( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("function") String function) { + LOG.info("Received drop function request: {}.{}.{}.{}", metalake, catalog, schema, function); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = + NameIdentifierUtil.ofFunction(metalake, catalog, schema, function); + boolean dropped = dispatcher.dropFunction(ident); + if (!dropped) { + LOG.warn("Cannot find to be dropped function {} under schema {}", function, schema); + } + Response response = Utils.ok(new DropResponse(dropped)); + LOG.info("Function dropped: {}.{}.{}.{}", metalake, catalog, schema, function); + return response; + }); + } catch (Exception e) { + return ExceptionHandlers.handleFunctionException(OperationType.DROP, function, schema, e); + } + } +} diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFunctionOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFunctionOperations.java new file mode 100644 index 00000000000..c1fafc096f6 --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFunctionOperations.java @@ -0,0 +1,804 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.FunctionDispatcher; +import org.apache.gravitino.dto.function.FunctionDefinitionDTO; +import org.apache.gravitino.dto.function.FunctionImplDTO; +import org.apache.gravitino.dto.function.FunctionParamDTO; +import org.apache.gravitino.dto.function.SQLImplDTO; +import org.apache.gravitino.dto.requests.FunctionRegisterRequest; +import org.apache.gravitino.dto.requests.FunctionUpdateRequest; +import org.apache.gravitino.dto.requests.FunctionUpdatesRequest; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ErrorConstants; +import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.FunctionListResponse; +import org.apache.gravitino.dto.responses.FunctionResponse; +import org.apache.gravitino.exceptions.FunctionAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchFunctionException; +import org.apache.gravitino.exceptions.NoSuchFunctionVersionException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.function.Function; +import org.apache.gravitino.function.FunctionChange; +import org.apache.gravitino.function.FunctionColumn; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionDefinitions; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; +import org.apache.gravitino.function.FunctionParam; +import org.apache.gravitino.function.FunctionParams; +import org.apache.gravitino.function.FunctionType; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; +import org.apache.gravitino.rest.RESTUtils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestFunctionOperations extends BaseOperationsTest { + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + private final FunctionDispatcher functionDispatcher = mock(FunctionDispatcher.class); + + private final AuditInfo testAuditInfo = + AuditInfo.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + private final String metalake = "metalake_for_function_test"; + + private final String catalog = "catalog_for_function_test"; + + private final String schema = "schema_for_function_test"; + + private final Namespace functionNs = NamespaceUtil.ofFunction(metalake, catalog, schema); + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(FunctionOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(functionDispatcher).to(FunctionDispatcher.class).ranked(2); + bindFactory(TestFunctionOperations.MockServletRequestFactory.class) + .to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testListFunctions() { + NameIdentifier funcId1 = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + NameIdentifier funcId2 = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func2"); + NameIdentifier[] funcIds = new NameIdentifier[] {funcId1, funcId2}; + when(functionDispatcher.listFunctions(functionNs)).thenReturn(funcIds); + + Response response = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + EntityListResponse resp = response.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp.getCode()); + Assertions.assertArrayEquals(funcIds, resp.identifiers()); + + // test listFunctions with details=true + Function mockFunction1 = mockFunction("func1", "comment1", FunctionType.SCALAR, 0); + Function mockFunction2 = mockFunction("func2", "comment2", FunctionType.SCALAR, 0); + Function[] functions = new Function[] {mockFunction1, mockFunction2}; + when(functionDispatcher.listFunctionInfos(functionNs)).thenReturn(functions); + + Response detailsResp = + target(functionPath()) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), detailsResp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, detailsResp.getMediaType()); + + FunctionListResponse funcListResp = detailsResp.readEntity(FunctionListResponse.class); + Assertions.assertEquals(0, funcListResp.getCode()); + Assertions.assertEquals(2, funcListResp.getFunctions().length); + Assertions.assertEquals("func1", funcListResp.getFunctions()[0].getName()); + Assertions.assertEquals("func2", funcListResp.getFunctions()[1].getName()); + + // Test mock return empty array for listFunctions + when(functionDispatcher.listFunctions(functionNs)).thenReturn(new NameIdentifier[0]); + Response resp3 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + EntityListResponse resp4 = resp3.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp4.getCode()); + Assertions.assertEquals(0, resp4.identifiers().length); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")) + .when(functionDispatcher) + .listFunctions(functionNs); + Response resp5 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(functionDispatcher).listFunctions(functionNs); + Response resp6 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp6.getStatus()); + + ErrorResponse errorResp1 = resp6.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetFunction() { + Function mockFunction = mockFunction("func1", "test comment", FunctionType.SCALAR, 0); + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + when(functionDispatcher.getFunction(funcId)).thenReturn(mockFunction); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + FunctionResponse funcResp = resp.readEntity(FunctionResponse.class); + Assertions.assertEquals(0, funcResp.getCode()); + + Function resultFunction = funcResp.getFunction(); + compare(mockFunction, resultFunction); + + // Test mock throw NoSuchFunctionException + doThrow(new NoSuchFunctionException("mock error")).when(functionDispatcher).getFunction(funcId); + Response resp1 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchFunctionException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(functionDispatcher).getFunction(funcId); + Response resp2 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetFunctionWithVersion() { + Function mockFunction = mockFunction("func1", "test comment", FunctionType.SCALAR, 1); + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + when(functionDispatcher.getFunction(funcId, 1)).thenReturn(mockFunction); + + Response resp = + target(functionPath()) + .path("func1") + .queryParam("version", 1) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + FunctionResponse funcResp = resp.readEntity(FunctionResponse.class); + Assertions.assertEquals(0, funcResp.getCode()); + Assertions.assertEquals(1, funcResp.getFunction().version()); + + // Test mock throw NoSuchFunctionVersionException + doThrow(new NoSuchFunctionVersionException("mock error")) + .when(functionDispatcher) + .getFunction(funcId, 1); + Response resp1 = + target(functionPath()) + .path("func1") + .queryParam("version", 1) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals( + NoSuchFunctionVersionException.class.getSimpleName(), errorResp.getType()); + + // Test a negative version + Response resp2 = + target(functionPath()) + .path("func1") + .queryParam("version", -1) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResp1.getCode()); + } + + @Test + public void testRegisterScalarFunction() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "test comment", FunctionType.SCALAR, 0); + + when(functionDispatcher.registerFunction( + eq(funcId), + anyString(), + eq(FunctionType.SCALAR), + anyBoolean(), + any(Type.class), + any(FunctionDefinition[].class))) + .thenReturn(mockFunction); + + FunctionDefinitionDTO[] definitionDTOs = createMockDefinitionDTOs(); + FunctionRegisterRequest req = + FunctionRegisterRequest.builder() + .name("func1") + .comment("test comment") + .functionType(FunctionType.SCALAR) + .deterministic(true) + .returnType(Types.IntegerType.get()) + .definitions(definitionDTOs) + .build(); + + Response resp = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + FunctionResponse funcResp = resp.readEntity(FunctionResponse.class); + Assertions.assertEquals(0, funcResp.getCode()); + compare(mockFunction, funcResp.getFunction()); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")) + .when(functionDispatcher) + .registerFunction( + eq(funcId), + anyString(), + eq(FunctionType.SCALAR), + anyBoolean(), + any(Type.class), + any(FunctionDefinition[].class)); + + Response resp1 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw FunctionAlreadyExistsException + doThrow(new FunctionAlreadyExistsException("mock error")) + .when(functionDispatcher) + .registerFunction( + eq(funcId), + anyString(), + eq(FunctionType.SCALAR), + anyBoolean(), + any(Type.class), + any(FunctionDefinition[].class)); + + Response resp2 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + FunctionAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(functionDispatcher) + .registerFunction( + eq(funcId), + anyString(), + eq(FunctionType.SCALAR), + anyBoolean(), + any(Type.class), + any(FunctionDefinition[].class)); + + Response resp3 = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testRegisterTableFunction() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "tableFunc1"); + Function mockFunction = mockTableFunction("tableFunc1", "test comment", 0); + + when(functionDispatcher.registerFunction( + eq(funcId), + anyString(), + anyBoolean(), + any(FunctionColumn[].class), + any(FunctionDefinition[].class))) + .thenReturn(mockFunction); + + FunctionRegisterRequest req = + FunctionRegisterRequest.builder() + .name("tableFunc1") + .comment("test comment") + .functionType(FunctionType.TABLE) + .deterministic(true) + .returnColumns(createMockReturnColumnDTOs()) + .definitions(createMockDefinitionDTOs()) + .build(); + + Response resp = + target(functionPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + FunctionResponse funcResp = resp.readEntity(FunctionResponse.class); + Assertions.assertEquals(0, funcResp.getCode()); + Assertions.assertEquals(FunctionType.TABLE, funcResp.getFunction().functionType()); + } + + @Test + public void testAlterFunction() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "new comment", FunctionType.SCALAR, 1); + + FunctionChange updateComment = FunctionChange.updateComment("new comment"); + when(functionDispatcher.alterFunction(funcId, updateComment)).thenReturn(mockFunction); + + FunctionUpdatesRequest req = + new FunctionUpdatesRequest( + Collections.singletonList( + new FunctionUpdateRequest.UpdateCommentRequest("new comment"))); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + FunctionResponse funcResp = resp.readEntity(FunctionResponse.class); + Assertions.assertEquals(0, funcResp.getCode()); + Assertions.assertEquals("new comment", funcResp.getFunction().comment()); + + // Test mock throw NoSuchFunctionException + doThrow(new NoSuchFunctionException("mock error")) + .when(functionDispatcher) + .alterFunction(funcId, updateComment); + + Response resp1 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchFunctionException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw IllegalArgumentException + doThrow(new IllegalArgumentException("mock error")) + .when(functionDispatcher) + .alterFunction(funcId, updateComment); + + Response resp2 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResp1.getCode()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(functionDispatcher) + .alterFunction(funcId, updateComment); + + Response resp3 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testAlterFunctionAddDefinition() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "comment", FunctionType.SCALAR, 1); + + FunctionDefinitionDTO newDef = createMockDefinitionDTOs()[0]; + FunctionChange addDef = FunctionChange.addDefinition(newDef.toFunctionDefinition()); + when(functionDispatcher.alterFunction(funcId, addDef)).thenReturn(mockFunction); + + FunctionUpdatesRequest req = + new FunctionUpdatesRequest( + Collections.singletonList(new FunctionUpdateRequest.AddDefinitionRequest(newDef))); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + } + + @Test + public void testAlterFunctionRemoveDefinition() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "comment", FunctionType.SCALAR, 1); + + FunctionParamDTO[] params = createMockParamDTOs(); + FunctionParam[] functionParams = new FunctionParam[params.length]; + for (int i = 0; i < params.length; i++) { + functionParams[i] = params[i].toFunctionParam(); + } + FunctionChange removeDef = FunctionChange.removeDefinition(functionParams); + when(functionDispatcher.alterFunction(funcId, removeDef)).thenReturn(mockFunction); + + FunctionUpdatesRequest req = + new FunctionUpdatesRequest( + Collections.singletonList(new FunctionUpdateRequest.RemoveDefinitionRequest(params))); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + } + + @Test + public void testAlterFunctionAddImpl() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "comment", FunctionType.SCALAR, 1); + + FunctionParamDTO[] params = createMockParamDTOs(); + FunctionParam[] functionParams = new FunctionParam[params.length]; + for (int i = 0; i < params.length; i++) { + functionParams[i] = params[i].toFunctionParam(); + } + FunctionImplDTO implDTO = createMockSqlImplDTO(); + FunctionImpl impl = implDTO.toFunctionImpl(); + FunctionChange addImpl = FunctionChange.addImpl(functionParams, impl); + when(functionDispatcher.alterFunction(funcId, addImpl)).thenReturn(mockFunction); + + FunctionUpdatesRequest req = + new FunctionUpdatesRequest( + Collections.singletonList(new FunctionUpdateRequest.AddImplRequest(params, implDTO))); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + } + + @Test + public void testAlterFunctionRemoveImpl() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + Function mockFunction = mockFunction("func1", "comment", FunctionType.SCALAR, 1); + + FunctionParamDTO[] params = createMockParamDTOs(); + FunctionParam[] functionParams = new FunctionParam[params.length]; + for (int i = 0; i < params.length; i++) { + functionParams[i] = params[i].toFunctionParam(); + } + FunctionChange removeImpl = + FunctionChange.removeImpl(functionParams, FunctionImpl.RuntimeType.SPARK); + when(functionDispatcher.alterFunction(funcId, removeImpl)).thenReturn(mockFunction); + + FunctionUpdatesRequest req = + new FunctionUpdatesRequest( + Collections.singletonList( + new FunctionUpdateRequest.RemoveImplRequest(params, "SPARK"))); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + } + + @Test + public void testDropFunction() { + NameIdentifier funcId = NameIdentifierUtil.ofFunction(metalake, catalog, schema, "func1"); + when(functionDispatcher.dropFunction(funcId)).thenReturn(true); + + Response resp = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for dropFunction + when(functionDispatcher.dropFunction(funcId)).thenReturn(false); + Response resp1 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(functionDispatcher).dropFunction(funcId); + Response resp2 = + target(functionPath()) + .path("func1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + private String functionPath() { + return "/metalakes/" + metalake + "/catalogs/" + catalog + "/schemas/" + schema + "/functions"; + } + + private Function mockFunction( + String name, String comment, FunctionType functionType, int version) { + Function mockFunction = mock(Function.class); + when(mockFunction.name()).thenReturn(name); + when(mockFunction.comment()).thenReturn(comment); + when(mockFunction.functionType()).thenReturn(functionType); + when(mockFunction.deterministic()).thenReturn(true); + when(mockFunction.returnType()).thenReturn(Types.IntegerType.get()); + when(mockFunction.returnColumns()).thenReturn(new FunctionColumn[0]); + when(mockFunction.definitions()).thenReturn(createMockDefinitions()); + when(mockFunction.version()).thenReturn(version); + when(mockFunction.auditInfo()).thenReturn(testAuditInfo); + return mockFunction; + } + + private Function mockTableFunction(String name, String comment, int version) { + Function mockFunction = mock(Function.class); + when(mockFunction.name()).thenReturn(name); + when(mockFunction.comment()).thenReturn(comment); + when(mockFunction.functionType()).thenReturn(FunctionType.TABLE); + when(mockFunction.deterministic()).thenReturn(true); + when(mockFunction.returnType()).thenReturn(null); + when(mockFunction.returnColumns()).thenReturn(createMockReturnColumns()); + when(mockFunction.definitions()).thenReturn(createMockDefinitions()); + when(mockFunction.version()).thenReturn(version); + when(mockFunction.auditInfo()).thenReturn(testAuditInfo); + return mockFunction; + } + + private FunctionDefinition[] createMockDefinitions() { + FunctionParam[] params = + new FunctionParam[] {FunctionParams.of("param1", Types.IntegerType.get())}; + FunctionImpl[] impls = + new FunctionImpl[] { + FunctionImpls.ofSql(FunctionImpl.RuntimeType.SPARK, "SELECT param1 + 1") + }; + return new FunctionDefinition[] {FunctionDefinitions.of(params, impls)}; + } + + private FunctionDefinitionDTO[] createMockDefinitionDTOs() { + return new FunctionDefinitionDTO[] { + FunctionDefinitionDTO.builder() + .withParameters(createMockParamDTOs()) + .withImpls(new FunctionImplDTO[] {createMockSqlImplDTO()}) + .build() + }; + } + + private FunctionParamDTO[] createMockParamDTOs() { + return new FunctionParamDTO[] { + FunctionParamDTO.builder().withName("param1").withDataType(Types.IntegerType.get()).build() + }; + } + + private FunctionImplDTO createMockSqlImplDTO() { + return new SQLImplDTO("SPARK", null, null, "SELECT param1 + 1"); + } + + private FunctionColumn[] createMockReturnColumns() { + return new FunctionColumn[] { + FunctionColumn.of("col1", Types.IntegerType.get(), "column comment") + }; + } + + private org.apache.gravitino.dto.function.FunctionColumnDTO[] createMockReturnColumnDTOs() { + return new org.apache.gravitino.dto.function.FunctionColumnDTO[] { + org.apache.gravitino.dto.function.FunctionColumnDTO.builder() + .withName("col1") + .withDataType(Types.IntegerType.get()) + .withComment("column comment") + .build() + }; + } + + private void compare(Function left, Function right) { + Assertions.assertEquals(left.name(), right.name()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.functionType(), right.functionType()); + Assertions.assertEquals(left.deterministic(), right.deterministic()); + Assertions.assertEquals(left.version(), right.version()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + } +}