add value support

README.md

@@ -1,8 +1,6 @@
 # OctoSQL Plugin for ETCD
 
-This is a plugin to run queries against etcd snapshots and raw database folders. This plugin is compatible with the key layout of Kubernetes.
-
-*Values are not supported (yet).*
+This is a plugin to run queries against etcd snapshots and raw database folders. This plugin is compatible with the key layout of Kubernetes and OpenShift.
 
 The very basic example is listing all keys:
 
@@ -27,9 +25,13 @@ $ octosql "SELECT * FROM etcdsnapshot.etcddump"
 This is also required when loading from a "dataDir" directly:
 
 ```sql
-$ octosql "SELECT * FROM etcdsnapshot./var/lib/etcd/"
+$ octosql "SELECT * FROM etcdsnapshot. /var/lib/etcd/"
 ```
 
+Mind the space and note that the database must be closed beforehand (i.e. etcd was properly shut down). 
+
+## Schema
+
 The table schema currently looks like that:
 
 ```sql
@@ -43,6 +45,7 @@ $ octosql "SELECT * FROM etcd.snapshot" --describe
 | 'name'            | 'NULL | String' | false      |
 | 'namespace'       | 'NULL | String' | false      |
 | 'resourceType'    | 'NULL | String' | false      |
+| 'value'           | 'String'        | false      |
 | 'valueSize'       | 'Int'           | false      |
 +-------------------+-----------------+------------+
 ```
@@ -53,12 +56,13 @@ $ octosql "SELECT * FROM etcd.snapshot" --describe
 * `resourceType` are the usual k8s resources like "pod", "service", "deployment"
 * `namespace` is the namespace of that resource
 * `name` is the resource name
+* `value` is the value as a string (usually JSON in K8s)
 * `valueSize` is the amount of bytes needed to store the value
 
 
 ## Examples
 
-Awesome queries you can run against your etcd snapshots now:
+Awesome queries you can run against your etcd (snapshots):
 
 ```sql
 $ octosql "SELECT COUNT(*) FROM etcd.snapshot"
@@ -115,7 +119,7 @@ $ octosql "SELECT namespace, COUNT(*) AS CNT FROM etcd.snapshot where resourceTy
 ### How many image streams are there?
 
 ```sql
-$ octosql "SELECT COUNT(*) AS CNT FROM etcd.snapshot where resourceType='imagestreams'  ORDER BY CNT DESC"
+$ octosql "SELECT COUNT(*) AS CNT FROM etcd.snapshot where resourceType='imagestreams' ORDER BY CNT DESC"
 +-----+
 | CNT |
 +-----+
@@ -158,6 +162,21 @@ $ octosql "SELECT namespace, SUM(valueSize) AS S from etcd.snapshot GROUP BY nam
   ....
 ```
 
+### Show me the value for the "version" resource
+
+```sql
+$ octosql "SELECT d.key, SUBSTR(d.value, 0, 10) FROM etcd.snapshot d WHERE name='version'"
++--------------------------------------------------------------+--------------+
+|                             key                              |    col_1     |
++--------------------------------------------------------------+--------------+
+| '/kubernetes.io/config.openshift.io/clusterversions/version' | '{"apiVersi' |
+| '/kubernetes.io/leases/openshift-cluster-version/version'    | ''           |
++--------------------------------------------------------------+--------------+
+```
+
+Note that "key" seems to be a reserved keyword, so when querying the key you will need to qualify with its table name.
+
+
 ## Installation
 
 1. Follow the instructions on [OctoSQL](https://github.com/cube2222/octosql) to install the query binary.

etcdsnapshot/execution.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	. "github.com/cube2222/octosql/execution"
 	"github.com/cube2222/octosql/octosql"
@@ -178,7 +179,12 @@ func mapEtcdToOctosql(kv mvccpb.KeyValue) []octosql.Value {
 		}
 	}
 
-	// add the size in bytes for the value, for easier sizing queries
-	values = append(values, octosql.NewInt(len(kv.Value)))
+	value := ""
+	if utf8.Valid(kv.Value) {
+		value = string(kv.Value)
+	}
+
+	// add the value and its size in bytes for the value, for easier sizing queries
+	values = append(values, octosql.NewString(value), octosql.NewInt(len(kv.Value)))
 	return values
 }

etcdsnapshot/execution_test.go

@@ -27,6 +27,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("some"),
 				octosql.NewInt(4),
 			},
 		},
@@ -40,6 +41,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("some"),
 				octosql.NewInt(4),
 			},
 		},
@@ -53,6 +55,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("some"),
 				octosql.NewInt(4),
 			},
 		},
@@ -66,6 +69,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewString("2"),
 				octosql.NewNull(),
 				octosql.NewString("3"),
+				octosql.NewString("some"),
 				octosql.NewInt(4),
 			},
 		},
@@ -79,6 +83,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewString("2"),
 				octosql.NewString("3"),
 				octosql.NewString("4"),
+				octosql.NewString("some-other"),
 				octosql.NewInt(10),
 			},
 		},
@@ -92,6 +97,7 @@ func TestMappingOfKeys(t *testing.T) {
 				octosql.NewString("3"),
 				octosql.NewString("4"),
 				octosql.NewString("5"),
+				octosql.NewString("some-other"),
 				octosql.NewInt(10),
 			},
 		},
@@ -109,7 +115,7 @@ func TestMappingOfKeys(t *testing.T) {
 func TestBasicSnapshot(t *testing.T) {
 	ds := &DatasourceExecuting{
 		path:         "data/basic.snapshot",
-		fieldIndices: []int{0, 1, 2, 3, 4, 5, 6},
+		fieldIndices: []int{0, 1, 2, 3, 4, 5, 6, 7},
 	}
 
 	expectedRecords := []execution.Record{
@@ -121,6 +127,7 @@ func TestBasicSnapshot(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("b"),
 				octosql.NewInt(1),
 			}, false, time.Time{}),
 
@@ -132,6 +139,7 @@ func TestBasicSnapshot(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("c"),
 				octosql.NewInt(1),
 			}, false, time.Time{}),
 		execution.NewRecord(
@@ -142,6 +150,7 @@ func TestBasicSnapshot(t *testing.T) {
 				octosql.NewNull(),
 				octosql.NewNull(),
 				octosql.NewNull(),
+				octosql.NewString("e"),
 				octosql.NewInt(1),
 			}, false, time.Time{}),
 	}
@@ -161,3 +170,24 @@ func TestBasicSnapshot(t *testing.T) {
 	require.NoError(t, err)
 	require.EqualValues(t, expectedRecords, records)
 }
+
+func _testFolder(t *testing.T) {
+	ds := &DatasourceExecuting{
+		path:         "/home/tjungblu/Downloads/ddump",
+		fieldIndices: []int{0, 1, 2, 3, 4, 5, 6, 7},
+	}
+
+	var records []execution.Record
+	err := ds.Run(execution.ExecutionContext{
+		Context:         context.TODO(),
+		VariableContext: nil,
+	},
+		func(ctx execution.ProduceContext, record execution.Record) error {
+			records = append(records, record)
+			return nil
+		},
+		nil,
+	)
+
+	require.NoError(t, err)
+}

etcdsnapshot/snapshot.go

@@ -37,7 +37,7 @@ func Creator(ctx context.Context, configUntyped plugins.ConfigDecoder) (physical
 }
 
 func (d Database) ListTables(ctx context.Context) ([]string, error) {
-	return []string{}, nil
+	return []string{"etcdsnapshot"}, nil
 }
 
 func (d Database) GetTable(ctx context.Context, name string, options map[string]string) (physical.DatasourceImplementation, physical.Schema, error) {
@@ -92,7 +92,10 @@ func (d Database) GetTable(ctx context.Context, name string, options map[string]
 			Name: "name",
 			Type: octosql.TypeSum(octosql.Null, octosql.String),
 		},
-
+		{
+			Name: "value",
+			Type: octosql.String,
+		},
 		// this should always be the last entry in this definition listing
 		{
 			Name: "valueSize",