Skip to content

Commit

Permalink
Merge branch 'master' of github.com:flyteorg/flytekit-java into sdkli…
Browse files Browse the repository at this point in the history
…teraltype-nesting-v2

Signed-off-by: Rafael Raposo <[email protected]>
  • Loading branch information
RRap0so committed Aug 2, 2024
2 parents 5757e97 + e647d77 commit 87f4fbc
Show file tree
Hide file tree
Showing 37 changed files with 1,341 additions and 304 deletions.
88 changes: 88 additions & 0 deletions flyteidl-protos/src/main/proto/flyteidl/admin/agent.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
syntax = "proto3";

package flyteidl.admin;
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";

import "flyteidl/core/literals.proto";
import "flyteidl/core/tasks.proto";
import "flyteidl/core/interface.proto";
import "flyteidl/core/identifier.proto";

// The state of the execution is used to control its visibility in the UI/CLI.
enum State {
RETRYABLE_FAILURE = 0;
PERMANENT_FAILURE = 1;
PENDING = 2;
RUNNING = 3;
SUCCEEDED = 4;
}

// Represents a subset of runtime task execution metadata that are relevant to external plugins.
message TaskExecutionMetadata {
// ID of the task execution
core.TaskExecutionIdentifier task_execution_id = 1;
// k8s namespace where the task is executed in
string namespace = 2;
// Labels attached to the task execution
map<string, string> labels = 3;
// Annotations attached to the task execution
map<string, string> annotations = 4;
// k8s service account associated with the task execution
string k8s_service_account = 5;
// Environment variables attached to the task execution
map<string, string> environment_variables = 6;
}

// Represents a request structure to create task.
message CreateTaskRequest {
// The inputs required to start the execution. All required inputs must be
// included in this map. If not required and not provided, defaults apply.
// +optional
core.LiteralMap inputs = 1;
// Template of the task that encapsulates all the metadata of the task.
core.TaskTemplate template = 2;
// Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
string output_prefix = 3;
// subset of runtime task execution metadata.
TaskExecutionMetadata task_execution_metadata = 4;
}

// Represents a create response structure.
message CreateTaskResponse {
// Metadata is created by the agent. It could be a string (jobId) or a dict (more complex metadata).
bytes resource_meta = 1;
}

// A message used to fetch a job resource from flyte agent server.
message GetTaskRequest {
// A predefined yet extensible Task type identifier.
string task_type = 1;
// Metadata about the resource to be pass to the agent.
bytes resource_meta = 2;
}

// Response to get an individual task resource.
message GetTaskResponse {
Resource resource = 1;
}

message Resource {
// The state of the execution is used to control its visibility in the UI/CLI.
State state = 1;
// The outputs of the execution. It's typically used by sql task. Agent service will create a
// Structured dataset pointing to the query result table.
// +optional
core.LiteralMap outputs = 2;
}

// A message used to delete a task.
message DeleteTaskRequest {
// A predefined yet extensible Task type identifier.
string task_type = 1;
// Metadata about the resource to be pass to the agent.
bytes resource_meta = 2;
}

// Response to delete a task.
message DeleteTaskResponse {
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,45 +6,6 @@ option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";

// Encapsulates specifications for routing an execution onto a specific cluster.
message ClusterAssignment {
Affinity affinity = 1;

Toleration toleration = 2;
}

// Defines a set of constraints used to select eligible objects based on labels they possess.
message Affinity {
// Multiples selectors are 'and'-ed together to produce the list of matching, eligible objects.
repeated Selector selectors = 1;
}

// Defines a set of specific label selectors that the execution can tolerate on a cluster.
message Toleration {

// A toleration selector is similar to that of an affinity but the only valid operators are EQUALS AND EXISTS.
repeated Selector selectors = 1;
reserved 1, 2;
string cluster_pool_name = 3;
}

// A Selector is a specification for identifying a set of objects with corresponding labels.
message Selector {

// The label key.
string key = 1;

// One or more values used to match labels.
// For equality (or inequality) requirements, values must contain a single element.
// For set-based requirements, values may contain one or more elements.
repeated string value = 2;

// Defines how a label with a corresponding key and value is selected or excluded.
enum Operator {
EQUALS = 0;
NOT_EQUALS = 1;
IN = 2;
NOT_IN = 3;
EXISTS = 4; // A label key with any value

// K8s supports more operators, we can consider adding them if necessary
}
Operator operator = 3;
}

17 changes: 17 additions & 0 deletions flyteidl-protos/src/main/proto/flyteidl/admin/common.proto
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";

import "flyteidl/core/execution.proto";
import "flyteidl/core/identifier.proto";
import "flyteidl/core/literals.proto";
import "google/protobuf/timestamp.proto";

// Encapsulation of fields that identifies a Flyte resource.
// A Flyte resource can be a task, workflow or launch plan.
Expand Down Expand Up @@ -279,6 +281,14 @@ message Annotations {
map<string, string> values = 1;
}

// Environment variable values to be applied to an execution resource.
// In the future a mode (e.g. OVERRIDE, APPEND, etc) can be defined
// to specify how to merge environment variables defined at registration and execution time.
message Envs {
// Map of custom environment variables to be applied to the execution resource.
repeated flyteidl.core.KeyValuePair values = 1;
}

// Defines permissions associated with executions created by this launch plan spec.
// Use either of these roles when they have permissions required by your workflow execution.
// Deprecated.
Expand All @@ -300,3 +310,10 @@ message RawOutputDataConfig {
// e.g. s3://bucket/key or s3://bucket/
string output_location_prefix = 1;
}

// These URLs are returned as part of node and task execution data requests.
message FlyteURLs {
string inputs = 1;
string outputs = 2;
string deck = 3;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
syntax = "proto3";

package flyteidl.admin;
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";

import "flyteidl/core/identifier.proto";
import "flyteidl/admin/common.proto";

// DescriptionEntity contains detailed description for the task/workflow.
// Documentation could provide insight into the algorithms, business use case, etc.
message DescriptionEntity {
// id represents the unique identifier of the description entity.
core.Identifier id = 1;
// One-liner overview of the entity.
string short_description = 2;
// Full user description with formatting preserved.
Description long_description = 3;
// Optional link to source code used to define this entity.
SourceCode source_code = 4;
// User-specified tags. These are arbitrary and can be used for searching
// filtering and discovering tasks.
repeated string tags = 5;
}

// The format of the long description
enum DescriptionFormat {
DESCRIPTION_FORMAT_UNKNOWN = 0;
DESCRIPTION_FORMAT_MARKDOWN = 1;
DESCRIPTION_FORMAT_HTML = 2;
// python default documentation - comments is rst
DESCRIPTION_FORMAT_RST = 3;
}

// Full user description with formatting preserved. This can be rendered
// by clients, such as the console or command line tools with in-tact
// formatting.
message Description {
oneof content {
// long description - no more than 4KB
string value = 1;
// if the description sizes exceed some threshold we can offload the entire
// description proto altogether to an external data store, like S3 rather than store inline in the db
string uri = 2;
}

// Format of the long description
DescriptionFormat format = 3;
// Optional link to an icon for the entity
string icon_link = 4;
}

// Link to source code used to define this entity
message SourceCode {
string link = 1;
}

// Represents a list of DescriptionEntities returned from the admin.
// See :ref:`ref_flyteidl.admin.DescriptionEntity` for more details
message DescriptionEntityList {
// A list of DescriptionEntities returned based on the request.
repeated DescriptionEntity descriptionEntities = 1;

// In the case of multiple pages of results, the server-provided token can be used to fetch the next page
// in a query. If there are no more results, this value will be empty.
string token = 2;
}

// Represents a request structure to retrieve a list of DescriptionEntities.
// See :ref:`ref_flyteidl.admin.DescriptionEntity` for more details
message DescriptionEntityListRequest {
// Identifies the specific type of resource that this identifier corresponds to.
flyteidl.core.ResourceType resource_type = 1;

// The identifier for the description entity.
// +required
NamedEntityIdentifier id = 2;

// Indicates the number of resources to be returned.
// +required
uint32 limit = 3;

// In the case of multiple pages of results, the server-provided token can be used to fetch the next page
// in a query.
// +optional
string token = 4;

// Indicates a list of filters passed as string.
// More info on constructing filters : <Link>
// +optional
string filters = 5;

// Sort ordering for returned list.
// +optional
Sort sort_by = 6;
}
38 changes: 38 additions & 0 deletions flyteidl-protos/src/main/proto/flyteidl/admin/execution.proto
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import "flyteidl/admin/common.proto";
import "flyteidl/core/literals.proto";
import "flyteidl/core/execution.proto";
import "flyteidl/core/identifier.proto";
import "flyteidl/core/metrics.proto";
import "flyteidl/core/security.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
Expand Down Expand Up @@ -45,10 +46,18 @@ message ExecutionRelaunchRequest {
// +required
core.WorkflowExecutionIdentifier id = 1;

// Deprecated field, do not use.
reserved 2;

// User provided value for the relaunched execution.
// If none is provided the system will generate a unique string.
// +optional
string name = 3;

// Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
// If enabled, all calculations are performed even if cached results would be available, overwriting the stored
// data once execution finishes successfully.
bool overwrite_cache = 4;
}

// Request to recover the referenced execution.
Expand Down Expand Up @@ -181,6 +190,9 @@ message SystemMetadata {

// Which execution cluster this execution ran on.
string execution_cluster = 1;

// Which kubernetes namespace the execution ran under.
string namespace = 2;
}

// Represents attributes about an execution which are not required to launch the execution but are useful to record.
Expand Down Expand Up @@ -296,6 +308,16 @@ message ExecutionSpec {
// As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
// around the bool field.
google.protobuf.BoolValue interruptible = 21;

// Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
// If enabled, all calculations are performed even if cached results would be available, overwriting the stored
// data once execution finishes successfully.
bool overwrite_cache = 22;

// Environment variables to be set for the execution.
Envs envs = 23;
// Tags to be set for the execution.
repeated string tags = 24;
}

// Request to terminate an in-progress execution. This action is irreversible.
Expand Down Expand Up @@ -368,3 +390,19 @@ message ExecutionStateChangeDetails {
}

message ExecutionUpdateResponse {}

// WorkflowExecutionGetMetricsRequest represents a request to retrieve metrics for the specified workflow execution.
message WorkflowExecutionGetMetricsRequest {
// id defines the workflow execution to query for.
core.WorkflowExecutionIdentifier id = 1;

// depth defines the number of Flyte entity levels to traverse when breaking down execution details.
int32 depth = 2;
}

// WorkflowExecutionGetMetricsResponse represents the response containing metrics for the specified workflow execution.
message WorkflowExecutionGetMetricsResponse {
// Span defines the top-level breakdown of the workflows execution. More precise information is nested in a
// hierarchical structure using Flyte entity references.
core.Span span = 1;
}
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ message LaunchPlanSpec {
// As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
// around the bool field.
google.protobuf.BoolValue interruptible = 19;

// Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
// If enabled, all calculations are performed even if cached results would be available, overwriting the stored
// data once execution finishes successfully.
bool overwrite_cache = 20;

// Environment variables to be set for the execution.
Envs envs = 21;
}

// Values computed by the flyte platform after launch plan registration.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,14 @@ message WorkflowExecutionConfig {
// As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
// around the bool field.
google.protobuf.BoolValue interruptible = 6;

// Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
// If enabled, all calculations are performed even if cached results would be available, overwriting the stored
// data once execution finishes successfully.
bool overwrite_cache = 7;

// Environment variables to be set for the execution.
Envs envs = 8;
}

// Generic container for encapsulating all types of the above attributes messages.
Expand Down
Loading

0 comments on commit 87f4fbc

Please sign in to comment.