Test.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;

using kinetica;

namespace Test
{
    public static class StringExt
    {
        public static string Truncate( this string value, int maxLength )
        {
            if ( string.IsNullOrEmpty( value ) )
                return value;

            return ( value.Length <= maxLength ) ? value : value.Substring( 0, maxLength );
        }
    }


    class Test
    {
        static void Main( string[] args )
        {
            Console.WriteLine( "Testing C# Project - Running" );
            Console.WriteLine();

            if ( args.Length < 1 )
            {
                Console.WriteLine( "Missing URL as command-line parameter." );
                Console.WriteLine( "E.g., http://kinetica:9191" );
                return;
            }

            try
            {
                string server_url = args[0];
                Console.WriteLine( "URL: {0}", server_url );
                Console.WriteLine();

                // Run various multihead ingest tests
                // ----------------------------------
                //test_authentication( server_url );
                test_all_types_5shard_0pk( server_url );
                test_all_types_3pk_3shard( server_url );
                test_all_types_3pk_2shard( server_url );
                test_all_types_3pk_1shard( server_url );
                test_all_types_3pk_0shard( server_url );
                test_all_types_0shard_0pk( server_url );
                test_all_types_ip_regex_3shard_0pk( server_url );
            }
            catch ( Exception ex )
            {
                Console.WriteLine( "Caught Exception: {0}", ex.Message );
            }

            // We're done
            Console.WriteLine();
            Console.WriteLine( "Testing C# Project - Done" );
            Console.WriteLine();
            Console.WriteLine( "Enter any 7-digit prime number to continue: " );
            Console.ReadLine();
        }  // end Main


        private class record_type_short
        {
            public int i { get; set; }
            public int i8 { get; set; }

            public override string ToString()
            {
                return $"{{ i={i}, i8={i8} }}";
            }
        }  // end class record_type_short


        private static void test_authentication( string server_url )
        {
            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Authentication" );
            Console.WriteLine( "===================" );
            Console.WriteLine();

            Console.WriteLine( "This test is done dynamically:" );
            Console.WriteLine( "------------------------------" );
            Console.WriteLine( "* First run--create the user with the 'correct' username" );
            Console.WriteLine( "* Second run--comment out user creation; run with 'correct'" );
            Console.WriteLine( "              username and make sure that the endpoints go through" );
            Console.WriteLine( "* Third run--comment out user creation; run with 'wrong'" );
            Console.WriteLine( "             username and make sure that we get an 'insufficient credentials'" );
            Console.WriteLine( "             error thrown from the server." );
            Console.WriteLine();

            string username = "abcdef"; // correct
            //string username = "abcdefg"; // wrong
            string password = "ghijkl123_";

            // Establish a connection with Kinetica
            // Create a user
            Kinetica kdb_ = new Kinetica( server_url );
            Console.WriteLine( $"Creating a user {username}" );
            Console.WriteLine();
            IDictionary <string, string> options = new Dictionary<string, string>();
            kdb_.createUserInternal( username, password, options );

            // Establish another connection with Kinetica with authentication information
            Console.WriteLine( "Creating a DB handle with the proper username and password" );
            Kinetica.Options db_options = new Kinetica.Options();
            db_options.Username = username;
            db_options.Password = password;
            Kinetica kdb = new Kinetica( server_url, db_options );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_short ) );

            // Create the type in the database
            string type_id = type1.create( kdb );

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();


            int num_records = 5000;
            kdb.insertRecordsRandom( table_name, num_records );
            Console.WriteLine( $"Generated {num_records} records." );
        }  // end test_authentication()


        private class record_type_all
        {
            public int i { get; set; }
            public int i8 { get; set; }
            public int i16 { get; set; }
            public long l { get; set; }
            public float f { get; set; }
            public double? d { get; set; } // nullable
            public string s { get; set; }
            public string c1 { get; set; }
            public string c2 { get; set; }
            public string c4 { get; set; }
            public string c8 { get; set; }
            public string c16 { get; set; }
            public string c32 { get; set; }
            public string c64 { get; set; }
            public string c128 { get; set; }
            public string c256 { get; set; }
            public string date { get; set; }
            public string decimal_ { get; set; }
            public string ipv4 { get; set; }
            public string time { get; set; }
            public long timestamp { get; set; }

            public override string ToString()
            {
                string d_;
                if ( d != null )
                    d_ = $"{d}";
                else
                    d_ = "<null>";
                return $"{{ i={i}, i8={i8},  i16={i16}, l={l}, f={f}, d={d_}, s={s}, c1={c1}, c2={c2}, c4={c4}, c8={c8}, c16={c16}, c32={c32}, c64={c64}, c128={c128}, c256={c256}, date={date}, decimal_={decimal_}, ipv4={ipv4}, time={time}, timestamp={timestamp} }}";
            }
        }  // end class record_type_all


        private static void test_all_types_ip_regex_3shard_0pk( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest with a Regex Given for Worker IP Addresses (3 shard keys, no PKs)" );
            Console.WriteLine( "=======================================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // Integer shard key
            List<string> i_props = new List<string>();
            i_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i", i_props );
            // Double shard key
            List<string> d_props = new List<string>();
            d_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "d", d_props );
            // Short shard key
            List<string> i16_props = new List<string>();
            i16_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i16", i16_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 100;
            Regex ip_regex = new Regex( "172.30.*" );
            KineticaIngestor<record_type_all>.WorkerList worker_list = new KineticaIngestor<record_type_all>.WorkerList( kdb, ip_regex );
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1, null, worker_list );

            // Generate data to be inserted
            int num_records = batch_size * 5;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_ip_regex_3shard_0pk()


        private static void test_all_types_0shard_0pk( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per type; 0 Shard Keys, 0 Primary Keys" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ) );

            // Create the type in the database
            string type_id = type1.create( kdb );

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_0shard_0pk()



        private static void test_all_types_5shard_0pk( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; 5 Shard Keys, 0 Primary Keys" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // Integer shard key
            List<string> i_props = new List<string>();
            i_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i", i_props );
            // Double shard key
            List<string> d_props = new List<string>();
            d_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "d", d_props );
            // Short shard key
            List<string> i16_props = new List<string>();
            i16_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "i16", i16_props );
            // char2 shard key
            List<string> c2_props = new List<string>();
            c2_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "c2", c2_props );
            // char8 shard key
            List<string> c8_props = new List<string>();
            c8_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "c8", c8_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_5shard_0pk()



        private static void test_all_types_3pk_3shard( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; 3 Primary Keys, 3 Shard Keys" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // long shard key, also primary key
            List<string> l_props = new List<string>();
            l_props.Add( ColumnProperty.PRIMARY_KEY );
            l_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "l", l_props );
            // date primary key as well as shard key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            date_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "date", date_props );
            // timestamp primary key as well as shard
            List<string> ipv4_props = new List<string>();
            ipv4_props.Add( ColumnProperty.PRIMARY_KEY );
            ipv4_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "ipv4", ipv4_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_3shard()



        private static void test_all_types_3pk_2shard( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; 3 Primary Keys, 2 Shard Keys" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // char2 shard key, also primary key
            List<string> c2_props = new List<string>();
            c2_props.Add( ColumnProperty.PRIMARY_KEY );
            c2_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "c2", c2_props );
            // date primary key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "date", date_props );
            // timestamp primary key as well as shard
            List<string> timestamp_props = new List<string>();
            timestamp_props.Add( ColumnProperty.PRIMARY_KEY );
            timestamp_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "timestamp", timestamp_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_2shard()



        private static void test_all_types_3pk_1shard( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per Type; 3 Primary Keys, 1 Shard Key" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // float shard key, also primary key
            List<string> f_props = new List<string>();
            f_props.Add( ColumnProperty.PRIMARY_KEY );
            f_props.Add( ColumnProperty.SHARD_KEY );
            column_properties.Add( "f", f_props );
            // date primary key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "date", date_props );
            // timestamp primary key as well as shard
            List<string> timestamp_props = new List<string>();
            timestamp_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "timestamp", timestamp_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_1shard()



        private static void test_all_types_3pk_0shard( string server_url )
        {
            // Establish a connection with Kinetica
            Kinetica kdb = new Kinetica( server_url );

            Console.WriteLine( "\n\n" );
            Console.WriteLine( "Test Multihead Ingest: One column per type; 3 Primary Keys, No Shard Key" );
            Console.WriteLine( "========================================================================" );
            Console.WriteLine();

            // Create a type for our record_type_1 class
            // Add some interesting properties for some of the data types
            IDictionary<string, IList<string>> column_properties = new Dictionary<string, IList<string>>();
            // float shard key, also primary key
            List<string> f_props = new List<string>();
            f_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "f", f_props );
            // date primary key
            List<string> date_props = new List<string>();
            date_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "date", date_props );
            // timestamp primary key as well as shard
            List<string> timestamp_props = new List<string>();
            timestamp_props.Add( ColumnProperty.PRIMARY_KEY );
            column_properties.Add( "timestamp", timestamp_props );

            // Create the KineticaType object which facilitates creating types in the database
            KineticaType type1 = KineticaType.fromClass( typeof( record_type_all ), column_properties );

            // Create the type in the database
            string type_id = type1.create( kdb );
            //Console.WriteLine( "ID of the created type: " + type_id );
            //Console.WriteLine();

            string table_name = "csharp_example_table_01";
            // Clear any previously made table with the same name
            Console.WriteLine( "Clearing any existing table named '{0}'", table_name );
            try { kdb.clearTable( table_name, null ); } catch ( Exception ex ) { /* I don't care if the table doesn't already exists! */ }
            Console.WriteLine();

            // Create a table of the given type
            Console.WriteLine( $"Creating table named '{table_name}'" );
            kdb.createTable( table_name, type_id );
            Console.WriteLine();

            // Create the ingestor
            int batch_size = 1000;
            KineticaIngestor<record_type_all> ingestor = new KineticaIngestor<record_type_all>( kdb, table_name, batch_size, type1 );

            // Generate data to be inserted
            int num_records = batch_size * 50;
            List<record_type_all> records = new List<record_type_all>();
            Random rng = new Random();
            double null_probability = 0.2;
            for ( int i = 0; i < num_records; ++i )
            {
                // Restricting string length to 256
                int max_str_len = rng.Next( 0, 256 );
                record_type_all record = new record_type_all()
                {
                    i = rng.Next(),
                    i8 = rng.Next(),
                    i16 = rng.Next(),
                    l  = (long)rng.Next(),
                    f  = (float)(rng.NextDouble() * rng.Next()),
                    d  = ( rng.NextDouble() < null_probability ) ? null : ( double? ) ( rng.NextDouble() * rng.Next() ),
                    s  = System.IO.Path.GetRandomFileName().Truncate( max_str_len ),
                    c1   = System.IO.Path.GetRandomFileName().Truncate( 1 ),
                    c2   = System.IO.Path.GetRandomFileName().Truncate( 2 ),
                    c4   = System.IO.Path.GetRandomFileName().Truncate( 4 ),
                    c8   = System.IO.Path.GetRandomFileName().Truncate( 8 ),
                    c16  = System.IO.Path.GetRandomFileName().Truncate( 16 ),
                    c32  = System.IO.Path.GetRandomFileName().Truncate( 32 ),
                    c64  = System.IO.Path.GetRandomFileName().Truncate( 64 ),
                    c128 = System.IO.Path.GetRandomFileName().Truncate( 128 ),
                    c256 = System.IO.Path.GetRandomFileName().Truncate( 256 ),
                    date = Test.generate_date( rng ),
                    decimal_ = Test.generate_decimal( rng ),
                    ipv4 = Test.generate_IPv4( rng ),
                    time = Test.generate_time( rng ),
                    timestamp = (long) rng.Next()
                };

                //Console.WriteLine( record.ToString() );  // debug~~~~~~~~
                records.Add( record );
            }  // end for loop

            Console.WriteLine( $"Generated {num_records} records." );

            // Insert the records into the ingestor
            Console.WriteLine( $"Inserting {num_records} records..." );
            ingestor.insert( records );

            // Flush the ingestor (which actually inserts the records)
            Console.WriteLine( "\nFlushing any remaining records." );
            ingestor.flush();
        }  // end test_all_types_3pk_0shard()



        public static string generate_date( Random rng )
        {
            int past_range   = -90 * 365; //980 years
            int future_range = 20 * 365; // 20 years
            int offset = rng.Next( past_range, future_range );
            DateTime random_date = DateTime.Today.AddDays( offset );
            return String.Format( "{0:yyyy-MM-dd}", random_date );
        }  // end generate_date



        private static readonly string numbers = "0123456789";

        public static string generate_decimal( Random rng )
        {
            string sign = (rng.Next( 2 ) == 0) ? "-" : "";
            string precision = new string( Enumerable.Repeat( '1', rng.Next( 20 ) ).Select( i => Test.numbers[ rng.Next( numbers.Length ) ] ).ToArray() );
            string scale = rng.Next( 0, 10000 ).ToString();
            return ( sign + precision + "." + scale );
        }  // end generate_decimal

        

        public static string generate_IPv4( Random rng )
        {
            string n1 = rng.Next( 256 ).ToString();
            string n2 = rng.Next( 256 ).ToString();
            string n3 = rng.Next( 256 ).ToString();
            string n4 = rng.Next( 256 ).ToString();
            string dot = ".";
            return ( n1 + dot + n2 + dot + n3 + dot + n4 );
        }  // end generate_ipv4



        public static string generate_time( Random rng )
        {
            int max_miliseconds = 86400000;
            DateTime time = DateTime.Today;
            //Console.WriteLine( time.ToLongTimeString() ); // debug~~~~~~~~
            time = time.Add( TimeSpan.FromMilliseconds( rng.Next( max_miliseconds ) ) );
            //Console.WriteLine( String.Format( "{0:HH}:{0:mm}:{0:ss}.{0:FFF}", time ) ); // debug~~~~~~
            string time_str;
            if ( rng.Next( 6 ) == 0 ) // one in five will get the milliseconds
                time_str = String.Format( "{0:HH}:{0:mm}:{0:ss}.{0:FFF}", time );
            else
                time_str = String.Format( "{0:HH}:{0:mm}:{0:ss}", time );
            return time_str;
        }  // end generate_time


    }  // end class Test
}  // end namespace Test