15 files changed, 872 insertions, 0 deletions
diff --git a/aclk/aclk-schemas/proto/aclk/v1/lib.proto b/aclk/aclk-schemas/proto/aclk/v1/lib.proto
new file mode 100644
index 000000000..f32c32c6e
--- /dev/null
+++ b/aclk/aclk-schemas/proto/aclk/v1/lib.proto
@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+package aclk_lib.v1;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "aclk_lib/v1;aclklib";
+
+// ACLKMessagePosition is used by sequenced messages to define their exact position
+message ACLKMessagePosition {
+  uint64 sequence_id = 1;
+  // auto generated in Agent's DB upon sequence_id creation
+  google.protobuf.Timestamp seq_id_created_at = 2;
+  uint64 previous_sequence_id = 3;
+}
+
+message Capability {
+  string name = 1;
+  uint32 version = 2;
+  // version == 0 is equivalent to not having the capability at all
+  bool enabled = 3;
+}
diff --git a/aclk/aclk-schemas/proto/agent/v1/cmds.proto b/aclk/aclk-schemas/proto/agent/v1/cmds.proto
new file mode 100644
index 000000000..c37c00c3a
--- /dev/null
+++ b/aclk/aclk-schemas/proto/agent/v1/cmds.proto
@@ -0,0 +1,79 @@
+syntax = "proto3";
+option go_package = "agent/v1;agent";
+
+package agent.v1;
+
+import "google/protobuf/timestamp.proto";
+import "proto/aclk/v1/lib.proto";
+
+message CancelPendingRequest {
+    // must match the ID sent with the request originally made
+    // other than this agent will not put conditions on it
+    // and will treat it as opaque string (it simply has to match)
+    // However this doesn't mean there are no conditions on the id
+    // made on the request side
+    string request_id = 1;
+
+    // time when the cancellation request was generated
+    google.protobuf.Timestamp timestamp = 2;
+
+    // optional might be useful for debugging purposes
+    string trace_id = 3;
+}
+
+// AgentCommand is sent from the Cloud to the Agent at `/agent/{claim_id}/inbound/v1/cmd/AgentCommand`
+// the message includes the resource that the Cloud needs to GET from the Agent HTTP API along with related metadata
+message AgentCommand {
+    // the topic to which the Cloud awaits for the AgentCommandResponse.
+    // example: `/svc/agent-data-ctrl/2d7b7edd-561e-4aec-8ac1-466a585520f5/resp`
+    string callback_topic = 1;
+    // the topic to which the Cloud awaits for the AgentCommandAck.
+    // example: `/svc/agent-data-ctrl/2d7b7edd-561e-4aec-8ac1-466a585520f5/resp`
+    string ack_topic = 2;
+    // unique identifier for the AgentCommand
+    // example: `617038b3-7c2a-4617-a78f-ab37bd820198`
+    string message_id = 3;
+    // defined in milliseconds, the time the Agent has to respond before Cloud
+    // considering the request as timed-out
+    // example: `60000`
+    uint64 timeout = 4;
+    // defined in milliseconds, the time the Agent has to send back to the Cloud
+    // an AgentCommandAck message signaling that is still working on the request
+    // example: `3000`
+    uint64 ack_timeout = 5;
+    // the requested Agent resource
+    // example: `/api/v2/data?query_params_go_here`
+    string resource = 6;
+}
+
+// AgentCommandAck is sent from the Agent to the Cloud at predefined intervals (`AgentCommand.ack_timeout`) to predefined topic (`AgentCommand.ack_topic`)
+// signaling that the Agent is still working to serve an AgentCommand (referenced by the message_id) that the Cloud sent
+message AgentCommandAck {
+    // unique identifier to reference AgentCommand on which the Agent is still working on serving
+    // example: `617038b3-7c2a-4617-a78f-ab37bd820198`
+    string message_id = 1;
+    // the timestamp when the Agent created this AgentCommandAck message
+    google.protobuf.Timestamp created_at = 2;
+    // integer revealing the progress of completion to serve the AgentCommand with the given message_id
+    // example: `25`
+    uint32 progress_percent = 3;
+}
+
+// AgentCommandResponse is sent from the Agent to the Cloud at `/agent/{claim_id}/inbound/v1/cmd/AgentCommand`
+// the message includes the resource that the Cloud needs to GET from the Agent HTTP API along with related metadata
+message AgentCommandResponse {
+    // unique identifier for the AgentCommand
+    // example: `617038b3-7c2a-4617-a78f-ab37bd820198`
+    string message_id = 1;
+    // the (http) status code of the Agent's API response
+    // example: `200`
+    uint32 status_code = 2;
+    // the dumped raw (http) response the Agent's API returned
+    bytes response = 3;
+    // the Agent's timestamp (aka legacy `timestamp`)
+    google.protobuf.Timestamp timestamp = 4;
+    // the timestamp when the Agent received the AgentCommand for execution (aka legacy `t-rx`)
+    google.protobuf.Timestamp received_at = 5;
+    // the amount of microseconds the Agent needed to execute the HTTP request of the AgentCommand (aka legacy`t-exec`)
+    uint64 exec_time = 6;
+}
diff --git a/aclk/aclk-schemas/proto/agent/v1/connection.proto b/aclk/aclk-schemas/proto/agent/v1/connection.proto
new file mode 100644
index 000000000..4321b0b90
--- /dev/null
+++ b/aclk/aclk-schemas/proto/agent/v1/connection.proto
@@ -0,0 +1,59 @@
+syntax = "proto3";
+option go_package = "agent/v1;agent";
+
+package agent.v1;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/duration.proto";
+import "proto/aclk/v1/lib.proto";
+
+message UpdateAgentConnection {
+    string claim_id = 1;
+    bool reachable = 2;
+
+    int64 session_id = 3;
+
+    ConnectionUpdateSource update_source = 4;
+
+    // mqtt_broker_addr shard to use for reaching the agent
+    // cloud injects this information
+    string mqtt_broker_addr = 5;
+
+    google.protobuf.Timestamp updated_at = 6;
+
+    // vmq_instance_id broker shard to use for reaching the agent
+    // cloud injects this information
+    int32 vmq_instance_id = 7;
+
+    // > 15 optional fields:
+    // How long the system was running until connection (only applicable when reachable=true)
+    google.protobuf.Duration system_uptime = 15;
+
+    // How long the netdata agent was running until connection (only applicable when reachable=true)
+    google.protobuf.Duration agent_uptime = 16;
+
+    repeated aclk_lib.v1.Capability capabilities = 17;
+}
+
+message SendNodeInstances {
+    string claim_id = 1;
+    Config config = 2;
+}
+
+// ConnectionUpdateSource is to determine whether the connection update was issued 
+enum ConnectionUpdateSource {
+    // CONNECTION_UPDATE_SOURCE_UNSPECIFIED acts as default value for protobuf and is never specified
+    CONNECTION_UPDATE_SOURCE_UNSPECIFIED = 0;
+    // CONNECTION_UPDATE_SOURCE_AGENT A direct message from an agent
+    CONNECTION_UPDATE_SOURCE_AGENT = 1;
+    // CONNECTION_UPDATE_SOURCE_LWT message delivered as the Last Will and Testiment  from MQTT broker if an agent connection with the broker is lost 
+    CONNECTION_UPDATE_SOURCE_LWT = 2;
+    // CONNECTION_UPDATE_SOURCE_HEURISTIC A cloud generated message to sanitize incorrect internal state
+    CONNECTION_UPDATE_SOURCE_HEURISTIC = 3;
+}
+
+message Config {
+    bool bearer_protection = 1;
+    bool cloud_only_notifications = 2;
+    bool universal_dashboards = 3;
+}
diff --git a/aclk/aclk-schemas/proto/agent/v1/disconnect.proto b/aclk/aclk-schemas/proto/agent/v1/disconnect.proto
new file mode 100644
index 000000000..852ef702a
--- /dev/null
+++ b/aclk/aclk-schemas/proto/agent/v1/disconnect.proto
@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package agent.v1;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "agent/v1;agent";
+
+// Sent by Cloud to instruct Agent to disconnect ASAP
+message DisconnectReq {
+  uint64 reconnect_after_seconds = 1;
+  bool permaban = 2;
+  google.protobuf.Timestamp created_at = 3;
+  uint32 error_code = 4;
+  string error_description = 5;
+}
diff --git a/aclk/aclk-schemas/proto/alarm/v1/config.proto b/aclk/aclk-schemas/proto/alarm/v1/config.proto
new file mode 100644
index 000000000..430078fcf
--- /dev/null
+++ b/aclk/aclk-schemas/proto/alarm/v1/config.proto
@@ -0,0 +1,61 @@
+syntax = "proto3";
+
+package alarms.v1;
+
+option go_package = "alarms/v1;alarms";
+
+message SendAlarmConfiguration{
+    string config_hash = 1;
+}
+
+message ProvideAlarmConfiguration {
+    string config_hash = 1;
+    AlarmConfiguration config = 2;
+}
+
+message AlarmConfiguration{
+    string alarm = 1;
+    string template = 2;
+    string on_chart = 3;
+    
+    string classification = 4;
+    string type = 5;
+    string component = 6;
+        
+    string os = 7;
+    string hosts = 8;
+    string plugin = 9;
+    string module = 10;
+    string charts = 11;
+    string families = 12;
+    string lookup = 13;
+    string every = 14;
+    string units = 15;
+
+    string green = 16;
+    string red = 17;
+
+    string calculation_expr = 18;
+    string warning_expr = 19;
+    string critical_expr = 20;
+    
+    string recipient = 21;
+    string exec = 22;
+    string delay = 23;
+    string repeat = 24;
+    string info = 25;
+    string options = 26;
+    string host_labels = 27;
+    
+    //parsed values from above config values
+    //indicated by p_
+    int32 p_db_lookup_after = 28;
+    int32 p_db_lookup_before = 29;
+    string p_db_lookup_dimensions = 30;
+    string p_db_lookup_method = 31;
+    string p_db_lookup_options = 32;
+    int32 p_update_every = 33;
+
+    string chart_labels = 34;
+    string summary = 35;
+}
diff --git a/aclk/aclk-schemas/proto/alarm/v1/stream.proto b/aclk/aclk-schemas/proto/alarm/v1/stream.proto
new file mode 100644
index 000000000..44f190682
--- /dev/null
+++ b/aclk/aclk-schemas/proto/alarm/v1/stream.proto
@@ -0,0 +1,148 @@
+syntax = "proto3";
+
+package alarms.v1;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "alarms/v1;alarms";
+
+message SendAlarmLogHealth {
+    string node_id = 1;
+}
+
+message AlarmLogHealth {
+    string claim_id = 1;
+    string node_id = 2;
+    bool enabled = 3;
+    AlarmLogStatus status = 4;
+    LogEntries log_entries = 5;
+}
+
+message LogEntries {
+    int64 first_sequence_id = 1;
+    google.protobuf.Timestamp first_when = 2;
+
+    int64 last_sequence_id = 3;
+    google.protobuf.Timestamp last_when = 4;
+}
+
+enum AlarmLogStatus {
+    ALARM_LOG_STATUS_UNSPECIFIED = 0;
+    ALARM_LOG_STATUS_RUNNING = 1;
+    ALARM_LOG_STATUS_IDLE = 2;
+}
+
+message StartAlarmStreaming {
+    string node_id = 1;
+    uint64 batch_id = 2 [deprecated=true];
+    uint64 start_sequnce_id = 3 [deprecated=true];
+    // Instructs the agent to sync all configured alarms
+    bool resets = 4;
+}
+
+message SendAlarmCheckpoint {
+    string node_id = 1;
+    string claim_id = 2;
+}
+
+message AlarmCheckpoint {
+    string node_id = 1;
+    string claim_id = 2;
+    bytes checksum = 3;
+}
+
+message AlarmLogEntry {
+    string node_id = 1;
+    string claim_id = 2;
+
+    // The chart's id field
+    string chart = 3;
+    string name = 4;
+    string family = 5;
+    uint64 batch_id = 6 [deprecated=true];
+    uint64 sequence_id = 7 [deprecated=true];
+    uint64 when = 8;
+
+    string config_hash = 9;
+
+    int32 utc_offset = 10;
+    string timezone = 11;
+
+    // Paths that can be custom for the same alarm, but depend on installation path for each user. Should be here or in config ?
+    string exec_path = 12;
+    string conf_source = 13;
+    string command = 14;
+
+    // In seconds, uint32 is safe ?
+    uint32 duration = 15;
+    uint32 non_clear_duration = 16;
+
+    AlarmStatus status = 17;
+    AlarmStatus old_status = 18;
+    uint64 delay = 19;
+    uint64 delay_up_to_timestamp = 20;
+    // Todo: verify that we need these. sequence_id doesn't suffice?
+    //  uint64 updated_by_id = 12;
+    //  uint64 updates_id = 13;
+    uint64 last_repeat = 21;
+    bool silenced = 22;
+
+    // Check if string values are needed
+    string value_string = 23;
+    string old_value_string = 24;
+
+    double value = 25;
+    double old_value = 26;
+
+    // Updated alarm entry, when the status of the alarm has been updated by a later entry
+    bool updated = 27;
+
+    // Rendered_info
+    string rendered_info = 28;
+
+    // The chart's context field
+    string chart_context = 29;
+
+    // Counter of alert transitions for this alert chain
+    uint64 event_id = 30;
+
+    // A unique uuid for this alert event
+    string transition_id = 31;
+
+    // The chart's name field
+    string chart_name = 32;
+
+    // The rendered summary
+    string summary = 33;
+}
+
+enum AlarmStatus {
+    ALARM_STATUS_NULL = 0;
+    ALARM_STATUS_UNKNOWN = 1;
+    ALARM_STATUS_REMOVED = 2;
+    ALARM_STATUS_NOT_A_NUMBER = 3;
+    ALARM_STATUS_CLEAR = 4;
+    ALARM_STATUS_WARNING = 5;
+    ALARM_STATUS_CRITICAL = 6;
+}
+
+// SendAlarmSnapshot: send from cloud to the agent, to initiate an AlarmSnapshot image of current alarms back to the cloud
+message SendAlarmSnapshot {
+    string node_id = 1;
+    string claim_id = 2;
+    uint64 snapshot_id = 3 [deprecated=true];
+    uint64 sequence_id = 4 [deprecated=true];
+    string snapshot_uuid = 5;
+}
+
+// Agent responds with AlarmSnapshot to a SendAlarmSnapshot message
+message AlarmSnapshot{
+    string node_id = 1;
+    string claim_id = 2;
+    uint64 snapshot_id = 3 [deprecated=true]; // Same id from SendAlarmSnapshot message
+    uint32 chunks = 4; // In case full snapshot can not fit in a single message, indicates the total number of messages for this snapshot_id
+    uint32 chunk_size = 5; // How many alerts this chunk contains
+    uint32 chunk = 6; // Chunk index of this message
+    repeated AlarmLogEntry alarms = 7; // a list of AlarmLogEntry's
+    string snapshot_uuid = 8;
+}
diff --git a/aclk/aclk-schemas/proto/chart/v1/config.proto b/aclk/aclk-schemas/proto/chart/v1/config.proto
new file mode 100644
index 000000000..f0c5e3a35
--- /dev/null
+++ b/aclk/aclk-schemas/proto/chart/v1/config.proto
@@ -0,0 +1,37 @@
+syntax = "proto3";
+
+package chart.v1;
+
+option go_package = "chart/config/v1;chartconfig";
+
+// UpdateChartConfigs command contains the list of missing chart configs from the cloud to agent
+message UpdateChartConfigs {
+  // claim_id, node_id pair is used to identify the Node Instance
+  string claim_id = 1;
+  string node_id = 2;
+  // list of config hashes missing from cloud and requested from the agent
+  repeated string config_hashes = 3;
+}
+
+message ChartConfigsUpdated {
+  repeated ChartConfigUpdated configs = 1;
+}
+
+message ChartConfigUpdated {
+  string type = 1;
+  string family = 2;
+  string context = 3;
+  string title = 4;
+  uint64 priority = 5;
+  string plugin = 6;
+  string module = 7;
+  ChartType chart_type = 8;
+  string units = 9;
+  string config_hash = 10;
+}
+
+enum ChartType {
+  LINE = 0;
+  AREA = 1;
+  STACKED = 2;
+}
diff --git a/aclk/aclk-schemas/proto/chart/v1/dimension.proto b/aclk/aclk-schemas/proto/chart/v1/dimension.proto
new file mode 100644
index 000000000..8bcb564b8
--- /dev/null
+++ b/aclk/aclk-schemas/proto/chart/v1/dimension.proto
@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package chart.v1;
+
+import "google/protobuf/timestamp.proto";
+
+import "proto/aclk/v1/lib.proto";
+
+option go_package = "chart/dimension/v1;chartdimension";
+
+// ChartDimensionUpdated is a single event sent from the Agent to the Cloud containing chart dimension data.
+//
+// ChartDimensionUpdated messages are dispatched in bulk to the Cloud wrapped in ChartsAndDimensionsUpdated messages.
+message ChartDimensionUpdated {
+  string id = 1;
+  string chart_id = 2;
+  string node_id = 3;
+  string claim_id = 4;
+  string name = 5;
+  google.protobuf.Timestamp created_at = 6;
+  // null value means that the dimension is currently collected (live)
+  google.protobuf.Timestamp last_timestamp = 7;
+  aclk_lib.v1.ACLKMessagePosition position = 8;
+}
diff --git a/aclk/aclk-schemas/proto/chart/v1/instance.proto b/aclk/aclk-schemas/proto/chart/v1/instance.proto
new file mode 100644
index 000000000..25c99e7c7
--- /dev/null
+++ b/aclk/aclk-schemas/proto/chart/v1/instance.proto
@@ -0,0 +1,32 @@
+syntax = "proto3";
+
+package chart.v1;
+
+import "proto/aclk/v1/lib.proto";
+
+option go_package = "chart/instance/v1;chartinstance";
+
+// ChartInstanceUpdated is a single event sent from the Agent to the Cloud containing chart instance data.
+//
+// ChartInstanceUpdated messages are dispatched in bulk to the Cloud wrapped in ChartsAndDimensionsUpdated messages.
+message ChartInstanceUpdated {
+  string id = 1;
+  string claim_id = 2;
+  string node_id = 3;
+  string name = 4;
+  map<string, string> chart_labels = 5;
+  MemoryMode memory_mode = 6;
+  // in seconds
+  uint32 update_every_interval = 7;
+  string config_hash = 8;
+  aclk_lib.v1.ACLKMessagePosition position = 9;
+}
+
+enum MemoryMode {
+  NONE = 0;
+  RAM = 1;
+  MAP = 2;
+  SAVE = 3;
+  ALLOC = 4;
+  DB_ENGINE = 5;
+}
diff --git a/aclk/aclk-schemas/proto/chart/v1/stream.proto b/aclk/aclk-schemas/proto/chart/v1/stream.proto
new file mode 100644
index 000000000..9473538f2
--- /dev/null
+++ b/aclk/aclk-schemas/proto/chart/v1/stream.proto
@@ -0,0 +1,86 @@
+syntax = "proto3";
+
+package chart.v1;
+
+import "google/protobuf/timestamp.proto";
+
+import "proto/chart/v1/instance.proto";
+import "proto/chart/v1/dimension.proto";
+
+option go_package = "chart/stream/v1;chartstream";
+
+// StreamChartsAndDimensions is a Command produced by the Cloud, consumed by the Agent.
+//
+// It instructs the Agent to start sending ChartsAndDimensionsUpdated messages for a NodeInstance
+// after the last sequence_id that the Cloud has successfully ingested.
+message StreamChartsAndDimensions {
+  // claim_id, node_id pair is used to identify the Node Instance
+  string claim_id = 1;
+  string node_id = 2;
+
+  // sequence_id last verified sequence sent by the Agent
+  uint64 sequence_id = 3;
+  // batch_id identifies the stream_id and gets incremented every time the Cloud sends a new StreamChartsAndDimensions command
+  uint64 batch_id = 4;
+  // seq_id_created_at autogenerated timestamp in Agent's DB upon sequence_id creation
+  google.protobuf.Timestamp seq_id_created_at = 5;
+}
+
+
+// ChartsAndDimensionsAck is an Event produced by the Cloud, consumed by the Agent.
+//
+// This Event is an acknowledgment from the Cloud side that Chart messages up to a specific last_sequence_id
+// have been successfully ingested, and could be potentially deleted from the Agent's DB.
+message ChartsAndDimensionsAck {
+  string claim_id = 1;
+  string node_id = 2;
+  // the last verified stored message's seq_id
+  uint64 last_sequence_id = 3;
+}
+
+// ResetChartMessages is a Command produced by the Agent, consumed by the Cloud.
+//
+// This Command instructs the Cloud to clear its Chart state for a specific NodeInstance and re-sync
+// because of a ResetReason.
+message ResetChartMessages {
+  // claim_id, node_id pair is used to identify the Node Instance
+  string claim_id = 1;
+  string node_id = 2;
+
+  ResetReason reason = 3;
+}
+
+enum ResetReason {
+  DB_EMPTY = 0;
+  SEQ_ID_NOT_EXISTS = 1;
+  TIMESTAMP_MISMATCH = 2;
+}
+
+// ChartsAndDimensionsUpdated is a wrapper Event (`fat` message) produced by the Agent, consumed by the Cloud.
+//
+// It potentially includes a collection of ChartInstanceUpdated messages and|or a collection of ChartDimensionUpdated messages.
+message ChartsAndDimensionsUpdated {
+  repeated chart.v1.ChartInstanceUpdated charts = 1;
+  repeated chart.v1.ChartDimensionUpdated dimensions = 2;
+  uint64 batch_id = 3;
+}
+
+// RetentionUpdated includes the available retentions (in seconds) of the dimensions - of a specific node instance and memory-mode -
+// on a per update_every level.
+// This message is sent over upon Agent Database rotation events to inform the Cloud in total about the newly updated data retentions
+// of a node instance's dimensions.
+message RetentionUpdated {
+  // claim_id, node_id pair is used to identify the Node Instance
+  string claim_id = 1;
+  string node_id = 2;
+  // the memory_mode used by the node instance's chart instances
+  chart.v1.MemoryMode memory_mode = 3;
+  // this mapping identifies the newly updated available retention (in seconds) of the node instance's dimensions
+  // the keys are the update_every categories of various dimensions (1, 2, 4, 10 etc.),
+  // and the values are the available retention (in seconds) of each dimension belonging to the update_every category
+  // denoted by the key
+  map<uint32, uint32> interval_durations = 4;
+  // the timestamp when the db rotation event took place. Can be used in conjunction with the interval_durations
+  // to compute the beginning of each `updated_every` group's retention
+  google.protobuf.Timestamp rotation_timestamp = 5;
+}
diff --git a/aclk/aclk-schemas/proto/context/v1/context.proto b/aclk/aclk-schemas/proto/context/v1/context.proto
new file mode 100644
index 000000000..eb771f8eb
--- /dev/null
+++ b/aclk/aclk-schemas/proto/context/v1/context.proto
@@ -0,0 +1,57 @@
+syntax = "proto3";
+
+package context.v1;
+
+option go_package = "context/v1;context";
+
+// ContextsUpdated is an Event produced by the Agent, consumed by the Cloud.
+//
+// it contains a collection of ContextUpdated messages for a specific NodeInstance.
+message ContextsUpdated {
+    // contexUpdates contains the collection of context updates
+    repeated ContextUpdated contextUpdates = 1;
+    // claim_id, node_id pair identifies the node instance
+    string claim_id = 2;
+    string node_id = 3;
+    // version_hash is the contexts version_hash result the cloud should
+    // get after applying this message updates.
+    uint64 version_hash = 4;
+    // it's and always increasing number to compare
+    // which version_hash is more recent between multiple
+    // ContextsUpdated messages. Bigger means more recent.
+    uint64 created_at = 5;
+}
+
+// ContextUpdated contains context data.
+message ContextUpdated {
+    // context id
+    string id = 1;
+    // context version is an epoch in seconds.
+    uint64 version = 2;
+    // first_entry, last_entry are epochs in seconds
+    uint64 first_entry = 3;
+    uint64 last_entry = 4;
+    // deleted flag is used to signal a context deletion
+    bool deleted = 5;
+    // context configuration fields
+    string title = 6;
+    uint64 priority = 7;
+    string chart_type = 8;
+    string units = 9;
+    string family = 10;
+}
+
+// ContextsSnapshot is an Event produced by the Agent, consumed by the Cloud.
+//
+// it contains a snapshot of the existing contexts on the Agent.
+// snapshot version and context versions are epochs in seconds so we can
+// identify if a context version was generated after a specific snapshot.
+message ContextsSnapshot {
+    // contexts contains the collection of existing contexts
+    repeated ContextUpdated contexts = 1;
+    // claim_id, node_id pair identifies the node instance
+    string claim_id = 2;
+    string node_id = 3;
+    // version is an epoch in seconds
+    uint64 version = 4;
+}
diff --git a/aclk/aclk-schemas/proto/context/v1/stream.proto b/aclk/aclk-schemas/proto/context/v1/stream.proto
new file mode 100644
index 000000000..a6e7e3abf
--- /dev/null
+++ b/aclk/aclk-schemas/proto/context/v1/stream.proto
@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package context.v1;
+
+option go_package = "context/v1;context";
+
+// ContextsCheckpoint is a Command produced by the Cloud, consumed by the Agent.
+//
+// It informs the Agent the contexts' version_hash that the cloud has for a specific NodeInstance.
+message ContextsCheckpoint {
+    // claim_id, node_id pair is used to identify the NodeInstance.
+    string claim_id = 1;
+    string node_id = 2;
+    // version_hash tells the Agent the current version hash for the contexts received
+    // if the version hash calculated by the Agent is different, Agent will request
+    // to re-sync all contexts.
+    uint64 version_hash= 3;
+}
+
+// StopStreamingContexts is a Command produced by the Cloud, consumed by the Agent.
+//
+// It instructs the Agent to stop sending ContextsUpdated messages for a NodeInstance
+// due to a reason.
+message StopStreamingContexts {
+    // claim_id, node_id pair is used to identify the node instance
+    string claim_id = 1;
+    string node_id = 2;
+    
+    StopStreamingContextsReason reason = 3;
+}
+
+enum StopStreamingContextsReason {
+  RATE_LIMIT_EXCEEDED = 0;
+}
diff --git a/aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto b/aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto
new file mode 100644
index 000000000..f0c02461e
--- /dev/null
+++ b/aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto
@@ -0,0 +1,37 @@
+syntax = "proto3";
+option go_package = "nodeinstance/connection/v1;nodeinstanceconnection";
+
+package nodeinstance.v1;
+
+import "google/protobuf/timestamp.proto";
+import "proto/aclk/v1/lib.proto";
+
+message UpdateNodeInstanceConnection {
+    string claim_id = 1;
+    string node_id = 2;
+
+    // liveness whether node data are actively streamed to the agent.
+    bool liveness = 3;
+
+    // queryable whether the agent has data about the node.
+    bool queryable = 4;
+
+    int64 session_id = 5;
+
+    google.protobuf.Timestamp updated_at = 6;
+
+    // mqtt_broker_addr shard to use for reaching the agent
+    // cloud injects this information.
+    string mqtt_broker_addr = 7;
+
+    // vmq_instance_id broker shard to use for reaching the agent
+    // cloud injects this information.
+    int32 vmq_instance_id = 8;
+
+    // hops is the number of streaming hops between collection of node data
+    // and the claimed agent. Zero if no streaming is involved.
+    int32 hops = 9;
+
+    // capabilities of node instance NOT the NODE or agent!!!
+    repeated aclk_lib.v1.Capability capabilities = 10;
+}
diff --git a/aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto b/aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto
new file mode 100644
index 000000000..922337154
--- /dev/null
+++ b/aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto
@@ -0,0 +1,32 @@
+syntax = "proto3";
+option go_package = "node_instance/creation/v1;node_instancecreation";
+
+package nodeinstance.create.v1;
+
+message CreateNodeInstance {
+    // Claim ID of the Agent the Node Instance belongs to.
+    // Eventually, the NodeInstance will be identified by the compilation of
+    // this claim_id and NodeID returned by `CreateNodeInstanceResult`
+    string claim_id = 1;
+    // Machine GUID of the Machine the request comes from
+    // Used to look for an existing NodeID in the space claim_id belongs to
+    string machine_guid = 2;
+    string hostname = 3;
+
+    // vmq_instance_id broker shard to use for reaching the agent
+    // cloud injects this information.
+    int32 vmq_instance_id = 4;
+    // mqtt_broker_addr shard to use for reaching the agent
+    // cloud injects this information.
+    string mqtt_broker_addr = 5;
+
+    // hops is the number of streaming hops between collection of node data
+    //   and the claimed agent. Zero if no streaming is involved.
+    int32 hops = 6;
+}
+
+message CreateNodeInstanceResult {
+    string node_id = 1;
+    string machine_guid = 2;
+}
+
diff --git a/aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto b/aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto
new file mode 100644
index 000000000..7aa9d0448
--- /dev/null
+++ b/aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto
@@ -0,0 +1,148 @@
+syntax = "proto3";
+option go_package = "node_instance/info/v1;nodeinstanceinfo";
+
+package nodeinstance.info.v1;
+
+import "google/protobuf/timestamp.proto";
+import "proto/aclk/v1/lib.proto";
+
+// UpdateNodeInfo (Command)
+//
+// pulsar topic: `UpdateNodeInfo` (sharded)
+//
+// key: `claim_id,node_id`
+//
+// Publishers: `netdata/agent`
+// Subscribers: `cloud-node-mqtt-output-service`
+//
+// When:
+//  On nodeinstance connect
+//
+message UpdateNodeInfo {
+  string node_id = 7;
+
+  string claim_id = 1;
+
+  NodeInfo data = 2;
+  // to be obsoleted in future
+  // all new fields should go into node_info
+  // or node_instance_info respectively
+
+  google.protobuf.Timestamp updated_at = 3;
+
+  int64 session_id = 4;
+
+  string machine_guid = 5;
+
+  bool child = 6;
+
+  MachineLearningInfo ml_info = 8;
+  // to be obsoleted in far future
+
+  NodeInfo2 node_info = 9;
+  // node_info shows data about actual node
+  // for example feature (ml) for this
+  // node (child) might be available/enabled on the node (child) directly
+  // but not available trough the parent (node_instance)
+
+  NodeInstanceInfo node_instance_info = 10;
+  // info specific to the node_instance for this node available trough agent
+  // who sends this message.
+  // e.g. machine learning is enabled for this node and processing is done
+  // by the actual agent (parent). (child itself might or might not be
+  // ml ml_capable by itself (see node_info))
+}
+
+message NodeInfo2 {
+  repeated  aclk_lib.v1.Capability capabilities = 1;
+}
+
+message NodeInstanceInfo {
+  repeated  aclk_lib.v1.Capability capabilities = 1;
+}
+
+// NodeInfo describes the metadata of a node
+message NodeInfo {
+  string name = 1;
+
+  string os = 2;
+  string os_name = 3;
+  string os_version = 4;
+
+  string kernel_name = 5;
+  string kernel_version = 6;
+
+  string architecture = 7;
+
+  // number of cpu cores in the node
+  uint32 cpus = 8;
+
+  // human readable (value + unit) frequency of cpu
+  string cpu_frequency = 9;
+
+  // human readable (value + unit) size of node's memory
+  string memory = 10;
+
+  // human readable (value + unit) size of all (sum) node's disks
+  string disk_space = 11;
+
+  // version of the netdata agent
+  string version = 12;
+
+  // release channel of netdata agent (example: nightly)
+  string release_channel = 13;
+
+  string timezone = 14;
+
+  // virtualization_type example: kvm (optional)
+  string virtualization_type = 15;
+
+  // container_type example: docker (optional)
+  string container_type = 16;
+
+  string custom_info = 17;
+
+  // [Obsolete] repeated string services = 18;
+  reserved 18;
+
+  string machine_guid = 19;
+
+  // [Obsolete] repeated MirroredHostStatus mirrored_hosts_status = 20;
+  reserved 20;
+
+  map<string, string> host_labels = 21;
+
+  MachineLearningInfo ml_info = 22;
+  
+  // [Obsolete] repeated string collectors = 23;
+  reserved 23;
+}
+
+message MachineLearningInfo {
+  // have ML capability
+  bool ml_capable = 1;
+
+  // runs ML functionality
+  bool ml_enabled = 2;
+}
+
+// UpdateNodeCollectors (Command)
+//
+// key: `claim_id,node_id`
+//
+// Publishers: `netdata/agent`
+//
+// When:
+//  On nodeinstance connect (after agent settles) and on detection of change of collectors
+//
+
+message CollectorInfo {
+  string module = 1;
+  string plugin = 2;
+}
+
+message UpdateNodeCollectors {
+  string claim_id = 1;
+  string node_id = 2;
+  repeated CollectorInfo collectors = 3;
+}