Test.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;

using kinetica;

namespace Test
{
    public static class StringExt
    {
        public static string Truncate( this string value, int maxLength )
        {
            if ( string.IsNullOrEmpty( value ) )
                return value;

            return ( value.Length <= maxLength ) ? value : value.Substring( 0, maxLength );
        }
    }


    class Test
    {
        static void Main( string[] args )
        {
            Console.WriteLine( "Testing C# Project - Running" );
            Console.WriteLine();

            if ( args.Length < 1 )
            {
                Console.WriteLine( "Missing URL as command-line parameter." );
                Console.WriteLine( "E.g., http://kinetica:9191" );
                return;
            }

            try
            {
                string server_url = args[0];
                Console.WriteLine( "URL: {0}", server_url );
                Console.WriteLine();

                Kinetica.Options _ServerOptions = new Kinetica.Options();
                _ServerOptions.Username = args[1];
                _ServerOptions.Password = args[2];


                // Run various multihead ingest tests
                // ----------------------------------
                test_authentication( server_url, _ServerOptions );
                test_all_types_allshard( server_url, _ServerOptions );
                test_all_types_3pk_3shard( server_url, _ServerOptions );
                test_all_types_ip_regex_3shard_0pk( server_url, _ServerOptions );
                test_all_types_3pk_0shard( server_url, _ServerOptions );
                test_record_retrieval_by_key( server_url, _ServerOptions );
            }
            catch ( kinetica.KineticaIngestor<record_type_all>.InsertException<record_type_all> ex )
            {
                Console.WriteLine( "Caught InsertException: {0}", ex.Message );
            }
            catch ( KineticaException ex )
            {
                Console.WriteLine( "Caught KineticaException: {0}", ex.Message );
            }
            catch ( Exception ex )
            {
                Console.WriteLine( "Caught Exception: {0}", ex.Message );
            }

            // We're done
            Console.WriteLine();
            Console.WriteLine( "Testing C# Project - Done" );
            Console.WriteLine();
            Console.WriteLine( "Enter any 7-digit prime number to continue: " );
            Console.ReadLine();
        }  // end Main


        private class record_type_short
        {
            public int i { get; set; }
            public int i8 { get; set; }

            public override string ToString()
            {
                return $"{{ i={i}, i8={i8} }}";
            }
        }  // end class record_type_short


        private static void test_authentication( string server_url, Kinetica.Options _ServerOptions )
        {
            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Authentication" );
            Console.WriteLine( "===================" );
            Console.WriteLine();

            Console.WriteLine( "This test is done dynamically:" );
            Console.WriteLine( "------------------------------" );
            Console.WriteLine( "* First run--create the user with the 'correct' username" );
            Console.WriteLine( "* Second run--comment out user creation; run with 'correct'" );
            Console.WriteLine( "              username and make sure that the endpoints go through" );
            Console.WriteLine( "* Third run--comment out user creation; run with 'wrong'" );
            Console.WriteLine( "             username and make sure that we get an 'insufficient credentials'" );
            Console.WriteLine( "             error thrown from the server." );
            Console.WriteLine();

            string username = "abcdef"; // correct
            //string username = "abcdefg"; // wrong
            string password = "ghijkl123_";

            // Establish a connection with Kinetica
            // Create a user
            Kinetica kdb_ = new Kinetica( server_url, _ServerOptions );
            Console.WriteLine( $"Creating a user {username}" );
            Console.WriteLine();
            IDictionary <string, string> options = new Dictionary<string, string>();
            try
            {
                kdb_.createUserInternal( username, password, options );
            }
            catch ( KineticaException ex )
            {
                Console.WriteLine( "Caught exception: " + ex.Message );
            }

            // Establish another connection with Kinetica with authentication information
            Console.WriteLine( "Creating a DB handle with the proper username and password" );
            Kinetica.Options db_options = new Kinetica.Options();
            db_options.Username = username;
            db_options.Password = password;
            Kinetica kdb = new Kinetica( server_url, db_options );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_short ) );

            // Create the type in the database
            string type_id = type1.create( kdb );

            string table_name = "csharp_example_table_01_auth";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();


            int num_records = 5000;
            kdb.insertRecordsRandom( table_name, num_records );
            Console.WriteLine( $"Generated {num_records} records." );
        }  // end test_authentication()


        private class record_type_all
        {
            public int? i { get; set; }
            public int? i8 { get; set; }
            public int? i16 { get; set; }
            public long l { get; set; }  // not making nullable here so that can be used as a primary key
            public float? f { get; set; }
            public double? d { get; set; } // nullable
            public string s { get; set; }
            public string c1 { get; set; }
            public string c2 { get; set; }
            public string c4 { get; set; }
            public string c8 { get; set; }
            public string c16 { get; set; }
            public string c32 { get; set; }
            public string c64 { get; set; }
            public string c128 { get; set; }
            public string c256 { get; set; }
            public string date { get; set; }
            public string datetime { get; set; }
            public string decimal_ { get; set; }
            public string ipv4 { get; set; }
            public string time { get; set; }
            public long? timestamp { get; set; }

            public override string ToString()
            {
                string d_;
                if ( d != null )
                    d_ = $"{d}";
                else
                    d_ = "<null>";
                return $"{{ i={i}, i8={i8},  i16={i16}, l={l}, f={f}, d={d_}, s={s}, c1={c1}, c2={c2}, c4={c4}, c8={c8}, c16={c16}, c32={c32}, c64={c64}, c128={c128}, c256={c256}, date={date}, datetime={datetime}, decimal_={decimal_}, ipv4={ipv4}, time={time}, timestamp={timestamp} }}";
            }
        }  // end class record_type_all


        private static void test_all_types_allshard( string server_url, Kinetica.Options _ServerOptions )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url, _ServerOptions );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; All are Shard Keys, 0 Primary Keys" );
            Console.WriteLine( "==============================================================================" );
            Console.WriteLine();

            Console.WriteLine( "Creating a type with all column types (all are nullable and shard columns).\n" );

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();

            // Integer shard key
            List<string> i_props = new List<string>();
            i_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i", i_props );

            // Integer8 shard key
            List<string> i8_props = new List<string>();
            i8_props.Add( ColumnProperty.SHARD_KEY );
            i8_props.Add( ColumnProperty.INT8 );
            column_properties.Add( "i8", i8_props );
            // Integer16 shard key
            List<string> i16_props = new List<string>();
            i16_props.Add( ColumnProperty.SHARD_KEY );
            i16_props.Add( ColumnProperty.INT16 );
            column_properties.Add( "i16", i16_props );

            // Double shard key
            List<string> d_props = new List<string>();
            d_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "d", d_props );

            // Float shard key
            List<string> f_props = new List<string>();
            f_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "f", f_props );

            // Long shard key
            List<string> l_props = new List<string>();
            l_props.Add( ColumnProperty.SHARD_KEY );
            l_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "l", l_props );

            // string nullable shard key
            List<string> s_props = new List<string>();
            s_props.Add( ColumnProperty.SHARD_KEY );
            s_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "s", s_props );

            // char1 shard key, nullable
            List<string> c1_props = new List<string>();
            c1_props.Add( ColumnProperty.SHARD_KEY );
            c1_props.Add( ColumnProperty.NULLABLE );
            c1_props.Add( ColumnProperty.CHAR1 );
            column_properties.Add( "c1", c1_props );

            // char2 shard key, nullable
            List<string> c2_props = new List<string>();
            c2_props.Add( ColumnProperty.SHARD_KEY );
            c2_props.Add( ColumnProperty.NULLABLE );
            c2_props.Add( ColumnProperty.CHAR2 );
            column_properties.Add( "c2", c2_props );

            // char4 shard key, nullable
            List<string> c4_props = new List<string>();
            c4_props.Add( ColumnProperty.SHARD_KEY );
            c4_props.Add( ColumnProperty.NULLABLE );
            c4_props.Add( ColumnProperty.CHAR4 );
            column_properties.Add( "c4", c4_props );

            // char8 shard key, nullable
            List<string> c8_props = new List<string>();
            c8_props.Add( ColumnProperty.SHARD_KEY );
            c8_props.Add( ColumnProperty.NULLABLE );
            c8_props.Add( ColumnProperty.CHAR8 );
            column_properties.Add( "c8", c8_props );

            // char16 shard key, nullable
            List<string> c16_props = new List<string>();
            c16_props.Add( ColumnProperty.SHARD_KEY );
            c16_props.Add( ColumnProperty.NULLABLE );
            c16_props.Add( ColumnProperty.CHAR16 );
            column_properties.Add( "c16", c16_props );

            // char32 shard key, nullable
            List<string> c32_props = new List<string>();
            c32_props.Add( ColumnProperty.SHARD_KEY );
            c32_props.Add( ColumnProperty.NULLABLE );
            c32_props.Add( ColumnProperty.CHAR32 );
            column_properties.Add( "c32", c32_props );

            // char64 shard key, nullable
            List<string> c64_props = new List<string>();
            c64_props.Add( ColumnProperty.SHARD_KEY );
            c64_props.Add( ColumnProperty.NULLABLE );
            c64_props.Add( ColumnProperty.CHAR64 );
            column_properties.Add( "c64", c64_props );

            // char128 shard key, nullable
            List<string> c128_props = new List<string>();
            c128_props.Add( ColumnProperty.SHARD_KEY );
            c128_props.Add( ColumnProperty.NULLABLE );
            c128_props.Add( ColumnProperty.CHAR128 );
            column_properties.Add( "c128", c128_props );

            // char256 shard key, nullable
            List<string> c256_props = new List<string>();
            c256_props.Add( ColumnProperty.SHARD_KEY );
            c256_props.Add( ColumnProperty.NULLABLE );
            c256_props.Add( ColumnProperty.CHAR256 );
            column_properties.Add( "c256", c256_props );

            // date shard key, nullable
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.DATE );
            date_props.Add( ColumnProperty.SHARD_KEY );
            date_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "date", date_props );

            // datetime shard key, nullable
            List<string> datetime_props = new List<string>();
            datetime_props.Add( ColumnProperty.DATETIME );
            datetime_props.Add( ColumnProperty.SHARD_KEY );
            datetime_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "datetime", datetime_props );

            // decimal shard key, nullable
            List<string> decimal_props = new List<string>();
            decimal_props.Add(ColumnProperty.DECIMAL);
            decimal_props.Add(ColumnProperty.SHARD_KEY);
            decimal_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "decimal_", decimal_props );

            // IPv4 shard key, nullable
            List<string> ipv4_props = new List<string>();
            ipv4_props.Add( ColumnProperty.IPV4 );
            ipv4_props.Add( ColumnProperty.SHARD_KEY );
            ipv4_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "ipv4", ipv4_props );

            // time shard key, nullable
            List<string> time_props = new List<string>();
            time_props.Add( ColumnProperty.TIME );
            time_props.Add( ColumnProperty.SHARD_KEY );
            time_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "time", time_props );

            // timestamp shard key, already nullable
            List<string> timestamp_props = new List<string>();
            timestamp_props.Add( ColumnProperty.TIMESTAMP );
            timestamp_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "timestamp", timestamp_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();
            Console.WriteLine( "Created type.\n" );

            string table_name = "csharp_example_table_all_shards";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'\n", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'\n" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );


            // Generate data to be inserted
            int num_records = batch_size * 10;
            Console.WriteLine( $"Starting to generate {num_records} records.\n" );
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.15;  // 15% chance of creating nulls
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i    = ( rng.NextDouble() < null_probability ) ? null : (int?) rng.Next(),
                    i8   = ( rng.NextDouble() < null_probability ) ? null : (int?) rng.Next( -128, 128 ), // upper exclusive
                    i16  = ( rng.NextDouble() < null_probability ) ? null : (int?) rng.Next( -32768, 32768), // upper exclusive
                    l    = (long)rng.Next(),
                    //l  = ( rng.NextDouble() < null_probability ) ? null : (long)rng.Next(),  // can't be made nullable if not declared as long?
                    f    = ( rng.NextDouble() < null_probability ) ? null : (float?)(rng.NextDouble() * rng.Next()),
                    d    = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s    = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 5) ), // any size between empty string to 4 length
                    c8   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 9) ), // any size between empty string to 8 length
                    c16  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 17) ), // any size between empty string to 16 length
                    c32  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 5, 33) ), // any size between [5, 32] in length
                    c64  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 10, 65) ), // any size between [10, 64] in length
                    c128 = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 10, 129) ), // any size between [10, 128] in length
                    c256 = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 25, 257) ), // any size between [25, 256] in length
                    date     = ( rng.NextDouble() < null_probability ) ? null : Test.generate_date( rng ),
                    datetime = (rng.NextDouble() < null_probability) ? null : Test.generate_datetime(rng),
                    decimal_ = (rng.NextDouble() < null_probability) ? null : Test.generate_decimal(rng),
                    ipv4     = ( rng.NextDouble() < null_probability ) ? null : Test.generate_IPv4( rng ),
                    time = (rng.NextDouble() < null_probability) ? null : Test.generate_time(rng),
                    timestamp = ( rng.NextDouble() < null_probability ) ? null : ((long?)rng.Next( -306102240, 293795424 ) * rng.Next( 100000 ))
                };

                //Console.WriteLine( "#" + i + ": " + record.ToString() + "\n" );  // debug~~~~~~~~
                //records.Add( record );
                ingestor.insert( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            //Console.WriteLine( $"Inserting {num_records} records..." );
            //ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_allshard()


        
        private static void test_record_retrieval_by_key( string server_url, Kinetica.Options _ServerOptions )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url, _ServerOptions );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Key Lookup: One column per Type; one numeric and one string" );
            Console.WriteLine( "shard key (no primary key)" );
            Console.WriteLine( "==========================================================================" );
            Console.WriteLine();

            Console.WriteLine( "Creating a type with all column types (all are nullable; one numeric and one string shard columns).\n" );

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();

            // Integer shard key
            List<string> i_props = new List<string>();
            column_properties.Add( "i", i_props );

            // Integer8 shard key
            List<string> i8_props = new List<string>();
            i8_props.Add( ColumnProperty.SHARD_KEY );
            i8_props.Add( ColumnProperty.INT8 );
            column_properties.Add( "i8", i8_props );
            // Integer16 shard key
            List<string> i16_props = new List<string>();
            i16_props.Add( ColumnProperty.INT16 );
            column_properties.Add( "i16", i16_props );

            // Double shard key
            List<string> d_props = new List<string>();
            column_properties.Add( "d", d_props );

            // Float shard key
            List<string> f_props = new List<string>();
            column_properties.Add( "f", f_props );

            // Long shard key
            List<string> l_props = new List<string>();
            l_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "l", l_props );

            // string nullable shard key
            List<string> s_props = new List<string>();
            s_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "s", s_props );

            // char1 shard key, nullable
            List<string> c1_props = new List<string>();
            c1_props.Add( ColumnProperty.SHARD_KEY );
            c1_props.Add( ColumnProperty.NULLABLE );
            c1_props.Add( ColumnProperty.CHAR1 );
            column_properties.Add( "c1", c1_props );

            // char2 shard key, nullable
            List<string> c2_props = new List<string>();
            c2_props.Add( ColumnProperty.NULLABLE );
            c2_props.Add( ColumnProperty.CHAR2 );
            column_properties.Add( "c2", c2_props );

            // char4 shard key, nullable
            List<string> c4_props = new List<string>();
            c4_props.Add( ColumnProperty.NULLABLE );
            c4_props.Add( ColumnProperty.CHAR4 );
            column_properties.Add( "c4", c4_props );

            // char8 shard key, nullable
            List<string> c8_props = new List<string>();
            c8_props.Add( ColumnProperty.NULLABLE );
            c8_props.Add( ColumnProperty.CHAR8 );
            column_properties.Add( "c8", c8_props );

            // char16 shard key, nullable
            List<string> c16_props = new List<string>();
            c16_props.Add( ColumnProperty.NULLABLE );
            c16_props.Add( ColumnProperty.CHAR16 );
            column_properties.Add( "c16", c16_props );

            // char32 shard key, nullable
            List<string> c32_props = new List<string>();
            c32_props.Add( ColumnProperty.NULLABLE );
            c32_props.Add( ColumnProperty.CHAR32 );
            column_properties.Add( "c32", c32_props );

            // char64 shard key, nullable
            List<string> c64_props = new List<string>();
            c64_props.Add( ColumnProperty.NULLABLE );
            c64_props.Add( ColumnProperty.CHAR64 );
            column_properties.Add( "c64", c64_props );

            // char128 shard key, nullable
            List<string> c128_props = new List<string>();
            c128_props.Add( ColumnProperty.NULLABLE );
            c128_props.Add( ColumnProperty.CHAR128 );
            column_properties.Add( "c128", c128_props );

            // char256 shard key, nullable
            List<string> c256_props = new List<string>();
            c256_props.Add( ColumnProperty.NULLABLE );
            c256_props.Add( ColumnProperty.CHAR256 );
            column_properties.Add( "c256", c256_props );

            // date shard key, nullable
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.DATE );
            date_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "date", date_props );

            // datetime shard key, nullable
            List<string> datetime_props = new List<string>();
            datetime_props.Add( ColumnProperty.DATETIME );
            datetime_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "datetime", datetime_props );

            // decimal shard key, nullable
            List<string> decimal_props = new List<string>();
            decimal_props.Add( ColumnProperty.DECIMAL );
            decimal_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "decimal_", decimal_props );

            // IPv4 shard key, nullable
            List<string> ipv4_props = new List<string>();
            ipv4_props.Add( ColumnProperty.IPV4 );
            ipv4_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "ipv4", ipv4_props );

            // time shard key, nullable
            List<string> time_props = new List<string>();
            time_props.Add( ColumnProperty.TIME );
            time_props.Add( ColumnProperty.NULLABLE );
            column_properties.Add( "time", time_props );

            // timestamp shard key, already nullable
            List<string> timestamp_props = new List<string>();
            timestamp_props.Add( ColumnProperty.TIMESTAMP );
            column_properties.Add( "timestamp", timestamp_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();
            Console.WriteLine( "Created type.\n" );

            string table_name = "csharp_example_table_all_shards";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'\n", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'\n" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create indices on the shard columns (6.2 limitation for key lookup)
            kdb.alterTable( table_name, "create_index", "i8" );
            kdb.alterTable( table_name, "create_index", "c1" );

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );


            // Generate data to be inserted
            // Note that the current sharded columns (i8 and c1) will not
            // get nulls since in 6.2, Kinetica does not support nulls for 
            // key lookup
            int num_records = batch_size * 10;
            Console.WriteLine( $"Starting to generate {num_records} records.\n" );
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.15;  // 15% chance of creating nulls
            double shard_null_probability = 0; // in 6.2, key lookup can't handle nulls
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i    = ( rng.NextDouble() < null_probability ) ? null : (int?) rng.Next(),
                    i8   = ( rng.NextDouble() < shard_null_probability ) ? null : (int?) rng.Next( 110, 128 ), // Limiting the range in order to get multiple records per shard key
                    //i8   = ( rng.NextDouble() < shard_null_probability ) ? null : (int?) rng.Next( -128, 128 ), // upper exclusive
                    i16  = ( rng.NextDouble() < null_probability ) ? null : (int?) rng.Next( -32768, 32768), // upper exclusive
                    l    = (long)rng.Next(),
                    //l  = ( rng.NextDouble() < null_probability ) ? null : (long)rng.Next(),  // can't be made nullable if not declared as long?
                    f    = ( rng.NextDouble() < null_probability ) ? null : (float?)(rng.NextDouble() * rng.Next()),
                    d    = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s    = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = ( rng.NextDouble() < shard_null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 5) ), // any size between empty string to 4 length
                    c8   = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 9) ), // any size between empty string to 8 length
                    c16  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 0, 17) ), // any size between empty string to 16 length
                    c32  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 5, 33) ), // any size between [5, 32] in length
                    c64  = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 10, 65) ), // any size between [10, 64] in length
                    c128 = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 10, 129) ), // any size between [10, 128] in length
                    c256 = ( rng.NextDouble() < null_probability ) ? null : System.IO.Path.GetRandomFileName().Truncate( rng.Next( 25, 257) ), // any size between [25, 256] in length
                    date     = ( rng.NextDouble() < null_probability ) ? null : Test.generate_date( rng ),
                    datetime = (rng.NextDouble() < null_probability) ? null : Test.generate_datetime(rng),
                    decimal_ = (rng.NextDouble() < null_probability) ? null : Test.generate_decimal(rng),
                    ipv4     = ( rng.NextDouble() < null_probability ) ? null : Test.generate_IPv4( rng ),
                    time = (rng.NextDouble() < null_probability) ? null : Test.generate_time(rng),
                    timestamp = ( rng.NextDouble() < null_probability ) ? null : ((long?)rng.Next( -306102240, 293795424 ) * rng.Next( 100000 ))
                };

                //Console.WriteLine( "#" + i + ": " + record.ToString() + "\n" );  // debug~~~~~~~~
                records.Add( record );
                ingestor.insert( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();

            // Look up records by any key
            Console.WriteLine("Creating the record retriever...");
            RecordRetriever<record_type_all> retriever = new RecordRetriever<record_type_all>( kdb, table_name, type1 );
            Console.WriteLine("Attempting record retrieval...");
            // Perform the key lookup operations
            GetRecordsResponse<record_type_all> response = retriever.getRecordsByKey( records[ 0 ] );
            Console.WriteLine( "Number of records fetched: {0}", response.total_number_of_records );
            Console.WriteLine( "Records fetched: ");
            foreach ( var record in response.data )
                Console.WriteLine( record.ToString() );

            Console.WriteLine( "Are there more records to fetch?: {0}", response.has_more_records);

        }  // end test_record_retrieval_by_key()


        private static void test_all_types_ip_regex_3shard_0pk( string server_url, Kinetica.Options _ServerOptions )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url, _ServerOptions );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest with a Regex Given for Worker IP Addresses (3 shard keys, no PKs)" );
            Console.WriteLine( "=======================================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // Integer shard key
            List<string> i_props = new List<string>();
            i_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i", i_props );
            // Double shard key
            List<string> d_props = new List<string>();
            d_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "d", d_props );
            // Short shard key
            List<string> i16_props = new List<string>();
            i16_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i16", i16_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );

            string table_name = "csharp_example_table_3s0pk";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 100;
            // Note: May need to adjust the regex for this test to pass
            Regex ip_regex = new Regex( "192.168.*" );
            kinetica.Utils.WorkerList worker_list = new kinetica.Utils.WorkerList( kdb, ip_regex );
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1, null, worker_list );

            // Generate data to be inserted
            int num_records = batch_size * 5;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    datetime = Test.generate_datetime( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_ip_regex_3shard_0pk()




        private static void test_all_types_3pk_3shard( string server_url, Kinetica.Options _ServerOptions )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url, _ServerOptions );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; 3 Primary Keys, 3 Shard Keys" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();

            // long shard key, also primary key
            List<string> l_props = new List<string>();
            l_props.Add( ColumnProperty.PRIMARY_KEY );
            l_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "l", l_props );

            // date primary key as well as shard key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.DATE );
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            date_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "date", date_props );

            // time primary key as well as shard
            List<string> time_props = new List<string>();
            time_props.Add( ColumnProperty.TIME );
            time_props.Add( ColumnProperty.PRIMARY_KEY );
            time_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "time", time_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_3s3pk";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 100;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    datetime = Test.generate_datetime( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                ingestor.insert( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_3shard()



        private static void test_all_types_3pk_0shard( string server_url, Kinetica.Options _ServerOptions )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url, _ServerOptions );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per type; 3 Primary Keys, No Shard Key" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();

            // long primary key
            List<string> l_props = new List<string>();
            l_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "l", l_props );

            // date primary key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "date", date_props );

            // time primary key as well as shard
            List<string> time_props = new List<string>();
            time_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "time", time_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_0s3pk";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 100;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    datetime = Test.generate_datetime( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                ingestor.insert( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_0shard()



        public static string generate_date( Random rng )
        {
            int past_range   = -90 * 365; //980 years
            int future_range = 20 * 365; // 20 years
            int offset = rng.Next( past_range, future_range );
            DateTime random_date = DateTime.Today.AddDays( offset );
            return String.Format( "{0:yyyy-MM-dd}", random_date );
        }  // end generate_date



        public static string generate_datetime( Random rng )
        {
            int past_range   = -90 * 365; //980 years
            int future_range = 20 * 365; // 20 years
            int offset  = rng.Next( past_range, future_range );
            int year    = rng.Next(1900, 2500);
            int month   = rng.Next(1, 12);
            int day     = rng.Next(1, 28);
            int hour    = rng.Next(0, 23);
            int minute  = rng.Next(0, 59);
            int second  = rng.Next(0, 59);
            int msecond = rng.Next(0, 999);

            DateTime random_datetime = new DateTime( year, month, day,
                                                     hour, minute, second, msecond );

            // We don't have to have any time component
            int time_chance_percent = 90;
            if (rng.Next(101) > time_chance_percent)
                return String.Format("{0:yyyy-MM-dd}", random_datetime);
            else
            {
                // Don't have to have the millisecond
                int num_ms_digits = rng.Next(7);
                switch (num_ms_digits)
                {
                    case 0:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss}", random_datetime);
                    case 1:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.F}", random_datetime);
                    case 2:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.FF}", random_datetime);
                    case 3:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.FFF}", random_datetime);
                    case 4:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.FFFF}", random_datetime);
                    case 5:
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.FFFFF}", random_datetime);
                    case 6:
                    default: // not necessary, but the stupid compiler complains
                        return String.Format("{0:yyyy-MM-dd HH:mm:ss.FFFFFF}", random_datetime);
                }
            }
        }  // end generate_datetime



        private static readonly string numbers = "0123456789";

        public static string generate_decimal( Random rng )
        {
            string sign = (rng.Next( 2 ) == 0) ? "-" : "";
            // Using up to 14 integral digits to keep things simple since the decimal
            // range is [-922337203685477.5808, 922337203685477.5807]
            string precision = new string( Enumerable.Repeat( '1', rng.Next( 15 ) ).Select( i => Test.numbers[ rng.Next( numbers.Length ) ] ).ToArray() );
            string scale = rng.Next( 0, 10000 ).ToString();
            return ( sign + precision + "." + scale );
        }  // end generate_decimal

        

        public static string generate_IPv4( Random rng )
        {
            string n1 = rng.Next( 256 ).ToString();
            string n2 = rng.Next( 256 ).ToString();
            string n3 = rng.Next( 256 ).ToString();
            string n4 = rng.Next( 256 ).ToString();
            string dot = ".";
            return ( n1 + dot + n2 + dot + n3 + dot + n4 );
        }  // end generate_ipv4



        public static string generate_time( Random rng )
        {
            int max_miliseconds = 86400000;
            DateTime time = DateTime.Today;
            //Console.WriteLine( time.ToLongTimeString() ); // debug~~~~~~~~
            time = time.Add( TimeSpan.FromMilliseconds( rng.Next( max_miliseconds ) ) );
            //Console.WriteLine( String.Format( "{0:HH}:{0:mm}:{0:ss}.{0:FFF}", time ) ); // debug~~~~~~
            string time_str;
            if ( (time.Millisecond > 0)  && (rng.Next( 1, 6 ) > 3) ) // two in five will get the milliseconds
                time_str = String.Format( "{0:HH}:{0:mm}:{0:ss}.{0:FFF}", time );
            else
                time_str = String.Format( "{0:HH}:{0:mm}:{0:ss}", time );
            return time_str;
        }  // end generate_time


    }  // end class Test
}  // end namespace Test