Kinetica   C#   API  Version 7.2.3.1
CreateTableExternal.cs
Go to the documentation of this file.
1 /*
2  * This file was autogenerated by the Kinetica schema processor.
3  *
4  * DO NOT EDIT DIRECTLY.
5  */
6 
7 using System.Collections.Generic;
8 
9 namespace kinetica;
10 
27 {
34  public struct CreateTableOptions
35  {
38  public const string TYPE_ID = "type_id";
39 
55  public const string NO_ERROR_IF_EXISTS = "no_error_if_exists";
56 
57  public const string TRUE = "true";
58  public const string FALSE = "false";
59 
76  public const string IS_REPLICATED = "is_replicated";
77 
83  public const string FOREIGN_KEYS = "foreign_keys";
84 
87  public const string FOREIGN_SHARD_KEY = "foreign_shard_key";
88 
129  public const string PARTITION_TYPE = "partition_type";
130 
134  public const string RANGE = "RANGE";
135 
139  public const string INTERVAL = "INTERVAL";
140 
144  public const string LIST = "LIST";
145 
149  public const string HASH = "HASH";
150 
154  public const string SERIES = "SERIES";
155 
161  public const string PARTITION_KEYS = "partition_keys";
162 
179  public const string PARTITION_DEFINITIONS = "partition_definitions";
180 
196  public const string IS_AUTOMATIC_PARTITION = "is_automatic_partition";
197 
201  public const string TTL = "ttl";
202 
205  public const string CHUNK_SIZE = "chunk_size";
206 
209  public const string CHUNK_COLUMN_MAX_MEMORY = "chunk_column_max_memory";
210 
213  public const string CHUNK_MAX_MEMORY = "chunk_max_memory";
214 
230  public const string IS_RESULT_TABLE = "is_result_table";
231 
235  public const string STRATEGY_DEFINITION = "strategy_definition";
236 
241  public const string COMPRESSION_CODEC = "compression_codec";
242  } // end struct CreateTableOptions
243 
247  public struct Options
248  {
256  public const string BAD_RECORD_TABLE_NAME = "bad_record_table_name";
257 
261  public const string BAD_RECORD_TABLE_LIMIT = "bad_record_table_limit";
262 
271  public const string BAD_RECORD_TABLE_LIMIT_PER_INPUT = "bad_record_table_limit_per_input";
272 
276  public const string BATCH_SIZE = "batch_size";
277 
292  public const string COLUMN_FORMATS = "column_formats";
293 
319  public const string COLUMNS_TO_LOAD = "columns_to_load";
320 
326  public const string COLUMNS_TO_SKIP = "columns_to_skip";
327 
350  public const string COMPRESSION_TYPE = "compression_type";
351 
353  public const string NONE = "none";
354 
356  public const string AUTO = "auto";
357 
359  public const string GZIP = "gzip";
360 
362  public const string BZIP2 = "bzip2";
363 
367  public const string DATASOURCE_NAME = "datasource_name";
368 
397  public const string DEFAULT_COLUMN_FORMATS = "default_column_formats";
398 
401  public const string DATALAKE_CATALOG = "datalake_catalog";
402 
404  public const string DATALAKE_PATH = "datalake_path";
405 
407  public const string DATALAKE_SNAPSHOT = "datalake_snapshot";
408 
436  public const string ERROR_HANDLING = "error_handling";
437 
440  public const string PERMISSIVE = "permissive";
441 
443  public const string IGNORE_BAD_RECORDS = "ignore_bad_records";
444 
449  public const string ABORT = "abort";
450 
471  public const string EXTERNAL_TABLE_TYPE = "external_table_type";
472 
475  public const string MATERIALIZED = "materialized";
476 
480  public const string LOGICAL = "logical";
481 
516  public const string FILE_TYPE = "file_type";
517 
519  public const string AVRO = "avro";
520 
523  public const string DELIMITED_TEXT = "delimited_text";
524 
526  public const string GDB = "gdb";
527 
529  public const string JSON = "json";
530 
532  public const string PARQUET = "parquet";
533 
535  public const string SHAPEFILE = "shapefile";
536 
553  public const string FLATTEN_COLUMNS = "flatten_columns";
554 
557  public const string TRUE = "true";
558 
561  public const string FALSE = "false";
562 
565  public const string GDAL_CONFIGURATION_OPTIONS = "gdal_configuration_options";
566 
590  public const string IGNORE_EXISTING_PK = "ignore_existing_pk";
591 
620  public const string INGESTION_MODE = "ingestion_mode";
621 
624  public const string FULL = "full";
625 
630  public const string DRY_RUN = "dry_run";
631 
636  public const string TYPE_INFERENCE_ONLY = "type_inference_only";
637 
641  public const string JDBC_FETCH_SIZE = "jdbc_fetch_size";
642 
646  public const string KAFKA_CONSUMERS_PER_RANK = "kafka_consumers_per_rank";
647 
650  public const string KAFKA_GROUP_ID = "kafka_group_id";
651 
665  public const string KAFKA_OFFSET_RESET_POLICY = "kafka_offset_reset_policy";
666 
667  public const string EARLIEST = "earliest";
668  public const string LATEST = "latest";
669 
684  public const string KAFKA_OPTIMISTIC_INGEST = "kafka_optimistic_ingest";
685 
690  public const string KAFKA_SUBSCRIPTION_CANCEL_AFTER = "kafka_subscription_cancel_after";
691 
694  public const string KAFKA_TYPE_INFERENCE_FETCH_TIMEOUT = "kafka_type_inference_fetch_timeout";
695 
697  public const string LAYER = "layer";
698 
747  public const string LOADING_MODE = "loading_mode";
748 
752  public const string HEAD = "head";
753 
762  public const string DISTRIBUTED_SHARED = "distributed_shared";
763 
781  public const string DISTRIBUTED_LOCAL = "distributed_local";
782 
784  public const string LOCAL_TIME_OFFSET = "local_time_offset";
785 
792  public const string MAX_RECORDS_TO_LOAD = "max_records_to_load";
793 
797  public const string NUM_TASKS_PER_RANK = "num_tasks_per_rank";
798 
806  public const string POLL_INTERVAL = "poll_interval";
807 
810  public const string PRIMARY_KEYS = "primary_keys";
811 
833  public const string REFRESH_METHOD = "refresh_method";
834 
839  public const string MANUAL = "manual";
840 
845  public const string ON_START = "on_start";
846 
849  public const string SCHEMA_REGISTRY_CONNECTION_RETRIES = "schema_registry_connection_retries";
850 
853  public const string SCHEMA_REGISTRY_CONNECTION_TIMEOUT = "schema_registry_connection_timeout";
854 
857  public const string SCHEMA_REGISTRY_MAX_CONSECUTIVE_CONNECTION_FAILURES = "schema_registry_max_consecutive_connection_failures";
858 
861  public const string MAX_CONSECUTIVE_INVALID_SCHEMA_FAILURE = "max_consecutive_invalid_schema_failure";
862 
865  public const string SCHEMA_REGISTRY_SCHEMA_NAME = "schema_registry_schema_name";
866 
869  public const string SHARD_KEYS = "shard_keys";
870 
873  public const string SKIP_LINES = "skip_lines";
874 
879  public const string START_OFFSETS = "start_offsets";
880 
894  public const string SUBSCRIBE = "subscribe";
895 
914  public const string TABLE_INSERT_MODE = "table_insert_mode";
915 
917  public const string SINGLE = "single";
918 
921  public const string TABLE_PER_FILE = "table_per_file";
922 
930  public const string TEXT_COMMENT_STRING = "text_comment_string";
931 
938  public const string TEXT_DELIMITER = "text_delimiter";
939 
954  public const string TEXT_ESCAPE_CHARACTER = "text_escape_character";
955 
969  public const string TEXT_HAS_HEADER = "text_has_header";
970 
980  public const string TEXT_HEADER_PROPERTY_DELIMITER = "text_header_property_delimiter";
981 
988  public const string TEXT_NULL_STRING = "text_null_string";
989 
1001  public const string TEXT_QUOTE_CHARACTER = "text_quote_character";
1002 
1010  public const string TEXT_SEARCH_COLUMNS = "text_search_columns";
1011 
1017  public const string TEXT_SEARCH_MIN_COLUMN_LENGTH = "text_search_min_column_length";
1018 
1032  public const string TRIM_SPACE = "trim_space";
1033 
1048  public const string TRUNCATE_STRINGS = "truncate_strings";
1049 
1064  public const string TRUNCATE_TABLE = "truncate_table";
1065 
1066  public const string TYPE_INFERENCE_MAX_RECORDS_READ = "type_inference_max_records_read";
1067 
1086  public const string TYPE_INFERENCE_MODE = "type_inference_mode";
1087 
1090  public const string ACCURACY = "accuracy";
1091 
1094  public const string SPEED = "speed";
1095 
1098  public const string REMOTE_QUERY = "remote_query";
1099 
1103  public const string REMOTE_QUERY_FILTER_COLUMN = "remote_query_filter_column";
1104 
1107  public const string REMOTE_QUERY_INCREASING_COLUMN = "remote_query_increasing_column";
1108 
1112  public const string REMOTE_QUERY_PARTITION_COLUMN = "remote_query_partition_column";
1113 
1132  public const string UPDATE_ON_EXISTING_PK = "update_on_existing_pk";
1133  } // end struct Options
1134 
1141  public string table_name { get; set; }
1142 
1168  public IList<string> filepaths { get; set; } = new List<string>();
1169 
1173  public IDictionary<string, IDictionary<string, string>> modify_columns { get; set; } = new Dictionary<string, IDictionary<string, string>>();
1174 
1432  public IDictionary<string, string> create_table_options { get; set; } = new Dictionary<string, string>();
1433 
2367  public IDictionary<string, string> options { get; set; } = new Dictionary<string, string>();
2368 
2372 
3598  IList<string> filepaths,
3599  IDictionary<string, IDictionary<string, string>> modify_columns = null,
3600  IDictionary<string, string> create_table_options = null,
3601  IDictionary<string, string> options = null)
3602  {
3603  this.table_name = table_name ?? "";
3604  this.filepaths = filepaths ?? new List<string>();
3605  this.modify_columns = modify_columns ?? new Dictionary<string, IDictionary<string, string>>();
3606  this.create_table_options = create_table_options ?? new Dictionary<string, string>();
3607  this.options = options ?? new Dictionary<string, string>();
3608  } // end constructor
3609 } // end class CreateTableExternalRequest
3610 
3615 {
3619  public string table_name { get; set; }
3620 
3624  public string type_id { get; set; }
3625 
3628  public string type_definition { get; set; }
3629 
3632  public string type_label { get; set; }
3633 
3636  public IDictionary<string, IList<string>> type_properties { get; set; } = new Dictionary<string, IList<string>>();
3637 
3639  public long count_inserted { get; set; }
3640 
3644  public long count_skipped { get; set; }
3645 
3648  public long count_updated { get; set; }
3649 
3651  public IDictionary<string, string> info { get; set; } = new Dictionary<string, string>();
3652 
3653  public IList<string> files { get; set; } = new List<string>();
3654 } // end class CreateTableExternalResponse
string type_label
The user-defined description associated with the table's structure.
const string HEAD
The head node loads all data.
CreateTableExternalRequest(string table_name, IList< string > filepaths, IDictionary< string, IDictionary< string, string >> modify_columns=null, IDictionary< string, string > create_table_options=null, IDictionary< string, string > options=null)
Constructs a CreateTableExternalRequest object with the specified parameters.
const string STRATEGY_DEFINITION
The tier strategy for the table and its columns.
const string SCHEMA_REGISTRY_CONNECTION_TIMEOUT
Confluent Schema registry connection timeout (in secs).
const string POLL_INTERVAL
If TRUE, the number of seconds between attempts to load external files into the table.
const string IS_REPLICATED
Affects the distribution scheme for the table's data.
const string PERMISSIVE
Records with missing columns are populated with nulls if possible; otherwise, the malformed records a...
const string CHUNK_COLUMN_MAX_MEMORY
Indicates the target maximum data size for each column in a chunk to be used for this table.
const string NUM_TASKS_PER_RANK
Number of tasks for reading file per rank.
const string TABLE_INSERT_MODE
Insertion scheme to use when inserting records from multiple shapefiles.
const string REMOTE_QUERY_PARTITION_COLUMN
Alias name for REMOTE_QUERY_FILTER_COLUMN.
A set of string constants for the parameter create_table_options.
const string EXTERNAL_TABLE_TYPE
Specifies whether the external table holds a local copy of the external data.
const string PARQUET
Apache Parquet file format.
const string SINGLE
Insert all records into a single table.
const string COMPRESSION_CODEC
The default compression codec for this table's columns.
const string TEXT_ESCAPE_CHARACTER
Specifies the character that is used to escape other characters in the source data.
const string DISTRIBUTED_SHARED
The head node coordinates loading data by worker processes across all nodes from shared files availab...
const string FOREIGN_SHARD_KEY
Foreign shard key of the format 'source_column references shard_by_column from target_table(primary_k...
const string DATALAKE_CATALOG
Name of an existing datalake(iceberg) catalog used in loading files.
const string FALSE
Reject new records when primary keys match existing records.
const string REFRESH_METHOD
Method by which the table can be refreshed from its source data.
const string TEXT_SEARCH_MIN_COLUMN_LENGTH
Set the minimum column size for strings to apply the 'text_search' property to.
const string TEXT_SEARCH_COLUMNS
Add 'text_search' property to internally inferenced string columns.
const string SHARD_KEYS
Comma separated list of column names to set as shard keys, when not specified in the type.
const string SCHEMA_REGISTRY_SCHEMA_NAME
Name of the Avro schema in the schema registry to use when reading Avro records.
const string KAFKA_TYPE_INFERENCE_FETCH_TIMEOUT
Maximum time to collect Kafka messages before type inferencing on the set of them.
KineticaData - class to help with Avro Encoding for Kinetica
Definition: KineticaData.cs:14
const string AUTO
Auto detect compression type.
const string PARTITION_DEFINITIONS
Comma-separated list of partition definitions, whose format depends on the choice of PARTITION_TYPE.
const string TABLE_PER_FILE
Insert records from each file into a new table corresponding to that file.
const string MAX_CONSECUTIVE_INVALID_SCHEMA_FAILURE
Max records to skip due to schema related errors, before failing.
const string IS_AUTOMATIC_PARTITION
If TRUE, a new partition will be created for values which don't fall into an existing partition.
const string FLATTEN_COLUMNS
Specifies how to handle nested columns.
const string COMPRESSION_TYPE
Source data compression type.
const string MAX_RECORDS_TO_LOAD
Limit the number of records to load in this request: if this number is larger than BATCH_SIZE,...
const string SCHEMA_REGISTRY_CONNECTION_RETRIES
Confluent Schema registry connection timeout (in secs).
string table_name
Value of table_name.
const string RANGE
Use range partitioning.
const string IS_RESULT_TABLE
Indicates whether the table is a memory-only table.
const string DEFAULT_COLUMN_FORMATS
Specifies the default format to be applied to source data loaded into columns with the corresponding ...
const string FOREIGN_KEYS
Semicolon-separated list of foreign keys, of the format '(source_column_name [, .....
const string SUBSCRIBE
Continuously poll the data source to check for new data and load it into the table.
const string MANUAL
Refresh only occurs when manually requested by invoking the refresh action of Kinetica....
const string DATALAKE_SNAPSHOT
Snapshot ID of datalake(iceberg) object.
const string LOGICAL
External data will not be loaded into the database; the data will be retrieved from the source upon s...
const string HASH
Use hash partitioning.
A set of results returned by Kinetica.createTableExternal.
const string TRIM_SPACE
If set to TRUE, remove leading or trailing space from fields.
const string TYPE_INFERENCE_ONLY
Infer the type of the source data and return, without ingesting any data.
const string KAFKA_OPTIMISTIC_INGEST
Enable optimistic ingestion where Kafka topic offsets and table data are committed independently to a...
IDictionary< string, string > options
Optional parameters.
const string MATERIALIZED
Loads a copy of the external data into the database, refreshed on demand.
long count_updated
[Not yet implemented] Number of records updated within the external table.
const string SHAPEFILE
ShapeFile file format.
const string CHUNK_SIZE
Indicates the number of records per chunk to be used for this table.
IList< string > filepaths
A list of file paths from which data will be sourced; For paths in KiFS, use the URI prefix of kifs:/...
const string FILE_TYPE
Specifies the type of the file(s) whose records will be inserted.
long count_skipped
Number of records skipped, when not running in ABORT error handling mode.
IDictionary< string, IDictionary< string, string > > modify_columns
Not implemented yet.
const string PRIMARY_KEYS
Comma separated list of column names to set as primary keys, when not specified in the type.
const string COLUMN_FORMATS
For each target column specified, applies the column-property-bound format to the source data loaded ...
const string SPEED
Scans data and picks the widest possible column types so that 'all' values will fit with minimum data...
const string CHUNK_MAX_MEMORY
Indicates the target maximum data size for all columns in a chunk to be used for this table.
const string KAFKA_OFFSET_RESET_POLICY
Policy to determine whether the Kafka data consumption starts either at earliest offset or latest off...
const string TYPE_INFERENCE_MODE
Optimize type inferencing for either speed or accuracy.
const string DATALAKE_PATH
Path of datalake(iceberg) object.
const string TYPE_ID
ID of a currently registered type.
const string COLUMNS_TO_LOAD
Specifies a comma-delimited list of columns from the source data to load.
const string JDBC_FETCH_SIZE
The JDBC fetch size, which determines how many rows to fetch per round trip.
long count_inserted
Number of records inserted into the external table.
const string KAFKA_CONSUMERS_PER_RANK
Number of Kafka consumer threads per rank (valid range 1-6).
const string DISTRIBUTED_LOCAL
A single worker process on each node loads all files that are available to it.
const string TRUNCATE_TABLE
If set to TRUE, truncates the table specified by table_name prior to loading the file(s).
const string NONE
No compression.
string type_definition
A JSON string describing the columns of the created external table.
const string ON_START
Refresh table on database startup and when manually requested by invoking the refresh action of Kinet...
const string UPDATE_ON_EXISTING_PK
Specifies the record collision policy for inserting into a table with a primary key.
const string LAYER
Geo files layer(s) name(s): comma separated.
const string BATCH_SIZE
Number of records to insert per batch when inserting data.
const string SKIP_LINES
Skip a number of lines from the beginning of the file.
const string IGNORE_EXISTING_PK
Specifies the record collision error-suppression policy for inserting into a table with a primary key...
const string TRUNCATE_STRINGS
If set to TRUE, truncate string values that are longer than the column's type size.
const string START_OFFSETS
Starting offsets by partition to fetch from kafka.
const string BAD_RECORD_TABLE_NAME
Name of a table to which records that were rejected are written.
const string IGNORE_BAD_RECORDS
Malformed records are skipped.
const string TEXT_NULL_STRING
Specifies the character string that should be interpreted as a null value in the source data.
const string TEXT_HAS_HEADER
Indicates whether the source data contains a header row.
const string GZIP
gzip file compression.
const string TRUE
Upsert new records when primary keys match existing records.
const string TEXT_DELIMITER
Specifies the character delimiting field values in the source data and field names in the header (if ...
const string GDB
Esri/GDB file format.
const string LOCAL_TIME_OFFSET
Apply an offset to Avro local timestamp columns.
const string REMOTE_QUERY
Remote SQL query from which data will be sourced.
CreateTableExternalRequest()
Constructs a CreateTableExternalRequest object with default parameters.
const string LIST
Use list partitioning.
const string COLUMNS_TO_SKIP
Specifies a comma-delimited list of columns from the source data to skip.
A set of string constants for the parameter options.
const string SCHEMA_REGISTRY_MAX_CONSECUTIVE_CONNECTION_FAILURES
Max records to skip due to SR connection failures, before failing.
const string REMOTE_QUERY_FILTER_COLUMN
Name of column to be used for splitting REMOTE_QUERY into multiple sub-queries using the data distrib...
const string ACCURACY
Scans data to get exactly-typed and sized columns for all data scanned.
IDictionary< string, IList< string > > type_properties
A mapping of each external table column name to an array of column properties associated with that co...
const string LOADING_MODE
Scheme for distributing the extraction and loading of data from the source data file(s).
const string INGESTION_MODE
Whether to do a full load, dry run, or perform a type inference on the source data.
const string REMOTE_QUERY_INCREASING_COLUMN
Column on subscribed remote query result that will increase for new records (e.g.,...
const string DRY_RUN
Does not load data, but walks through the source data and determines the number of valid records,...
const string KAFKA_SUBSCRIPTION_CANCEL_AFTER
Sets the Kafka subscription lifespan (in minutes).
const string PARTITION_KEYS
Comma-separated list of partition keys, which are the columns or column expressions by which records ...
const string BZIP2
bzip2 file compression.
const string AVRO
Avro file format.
string type_id
ID of the currently registered table structure type for this external table.
const string GDAL_CONFIGURATION_OPTIONS
Comma separated list of gdal conf options, for the specific requests: key=value.
const string TEXT_QUOTE_CHARACTER
Specifies the character that should be interpreted as a field value quoting character in the source d...
const string TTL
Sets the TTL of the table specified in table_name.
const string TEXT_HEADER_PROPERTY_DELIMITER
Specifies the delimiter for column properties in the header row (if present).
const string SERIES
Use series partitioning.
const string ABORT
Stops current insertion and aborts entire operation when an error is encountered.
const string DELIMITED_TEXT
Delimited text file format; e.g., CSV, TSV, PSV, etc.
const string FULL
Run a type inference on the source data (if needed) and ingest.
const string PARTITION_TYPE
Partitioning scheme to use.
string table_name
Name of the table to be created, in [schema_name.
const string DATASOURCE_NAME
Name of an existing external data source from which data file(s) specified in filepaths will be loade...
const string ERROR_HANDLING
Specifies how errors should be handled upon insertion.
const string TEXT_COMMENT_STRING
Specifies the character string that should be interpreted as a comment line prefix in the source data...
IDictionary< string, string > create_table_options
Options from Kinetica.createTable, allowing the structure of the table to be defined independently of...
const string BAD_RECORD_TABLE_LIMIT
A positive integer indicating the maximum number of records that can be written to the bad-record-tab...
const string BAD_RECORD_TABLE_LIMIT_PER_INPUT
For subscriptions, a positive integer indicating the maximum number of records that can be written to...
A set of parameters for Kinetica.createTableExternal.
const string INTERVAL
Use interval partitioning.
IDictionary< string, string > info
Additional information.
const string KAFKA_GROUP_ID
The group id to be used when consuming data from a Kafka topic (valid only for Kafka datasource subsc...
const string JSON
JSON file format.
const string NO_ERROR_IF_EXISTS
If TRUE, prevents an error from occurring if the table already exists and is of the given type.