Add the ability to post to and act uponconfiguration in etcd

Etcd is a distributed, highly available name-value store that is
the basis of many clustered applications. However, it can also be
run on a standalone machine. Using etcd allows live monitoring
and possibly changing configuration while the sim is running.

OpenSim/Region/Framework/Interfaces/IEtcdModule.cs

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+
+public interface IEtcdModule
+{
+    bool Store(string k, string v);
+    bool Store(string k, string v, int ttl);
+    string Get(string k);
+    void Watch(string k, Action<string> callback);
+    void Delete(string k);
+}

OpenSim/Region/Framework/Scenes/Scene.cs

@@ -540,6 +540,9 @@ namespace OpenSim.Region.Framework.Scenes
         private Timer m_mapGenerationTimer = new Timer();
         private bool m_generateMaptiles;
 
+        protected int m_lastHealth = -1;
+        protected int m_lastUsers = -1;
+
         #endregion Fields
 
         #region Properties
@@ -1234,6 +1237,8 @@ namespace OpenSim.Region.Framework.Scenes
                     }
                 }
             }
+
+            StartTimerWatchdog();
         }
 
         public Scene(RegionInfo regInfo)
@@ -1504,6 +1509,14 @@ namespace OpenSim.Region.Framework.Scenes
                 return;
             }
 
+            IEtcdModule etcd = RequestModuleInterface<IEtcdModule>();
+            if (etcd != null)
+            {
+                etcd.Delete("Health");
+                etcd.Delete("HealthFlags");
+                etcd.Delete("RootAgents");
+            }
+
             m_log.InfoFormat("[SCENE]: Closing down the single simulator: {0}", RegionInfo.RegionName);
 
 
@@ -5536,23 +5549,24 @@ Label_GroupsDone:
                 return 0;
             }
 
-            if ((Util.EnvironmentTickCountSubtract(m_lastFrameTick)) < 1000)
+            if ((Util.EnvironmentTickCountSubtract(m_lastFrameTick)) < 2000)
             {
                 health+=1;
                 flags |= 1;
             }
 
-            if (Util.EnvironmentTickCountSubtract(m_lastIncoming) < 1000)
+            if (Util.EnvironmentTickCountSubtract(m_lastIncoming) < 2000)
             {
                 health+=1;
                 flags |= 2;
             }
 
-            if (Util.EnvironmentTickCountSubtract(m_lastOutgoing) < 1000)
+            if (Util.EnvironmentTickCountSubtract(m_lastOutgoing) < 2000)
             {
                 health+=1;
                 flags |= 4;
             }
+            /*
             else
             {
 int pid = System.Diagnostics.Process.GetCurrentProcess().Id;
@@ -5565,6 +5579,7 @@ proc.WaitForExit();
 Thread.Sleep(1000);
 Environment.Exit(1);
             }
+            */
 
             if (flags != 7)
                 return health;
@@ -6329,6 +6344,32 @@ Environment.Exit(1);
         public void TimerWatchdog(object sender, ElapsedEventArgs e)
         {
             CheckHeartbeat();
+
+            IEtcdModule etcd = RequestModuleInterface<IEtcdModule>();
+            int flags;
+            string message;
+            if (etcd != null)
+            {
+                int health = GetHealth(out flags, out message);
+                if (health != m_lastHealth)
+                {
+                    m_lastHealth = health;
+
+                    etcd.Store("Health", health.ToString(), 300000);
+                    etcd.Store("HealthFlags", flags.ToString(), 300000);
+                }
+
+                int roots = 0;
+                foreach (ScenePresence sp in GetScenePresences())
+                    if (!sp.IsChildAgent && !sp.IsNPC)
+                        roots++;
+
+                if (m_lastUsers != roots)
+                {
+                    m_lastUsers = roots;
+                    etcd.Store("RootAgents", roots.ToString(), 300000);
+                }
+            }
         }
 
         /// This method deals with movement when an avatar is automatically moving (but this is distinct from the

OpenSim/Region/OptionalModules/Framework/Monitoring/EtcdMonitoringModule.cs

@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text;
+using log4net;
+using Mono.Addins;
+using Nini.Config;
+using OpenMetaverse;
+using OpenSim.Framework;
+using OpenSim.Framework.Console;
+using OpenSim.Region.Framework.Interfaces;
+using OpenSim.Region.Framework.Scenes;
+using netcd;
+using netcd.Serialization;
+using netcd.Advanced;
+using netcd.Advanced.Requests;
+
+namespace OpenSim.Region.OptionalModules.Framework.Monitoring
+{
+    /// <summary>
+    /// Allows to store monitoring data in etcd, a high availability
+    /// name-value store.
+    /// </summary>
+    [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "EtcdMonitoringModule")]
+    public class EtcdMonitoringModule : INonSharedRegionModule, IEtcdModule
+    {
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        protected Scene m_scene;
+        protected IEtcdClient m_client;
+        protected bool m_enabled = false;
+        protected string m_etcdBasePath = String.Empty;
+        protected bool m_appendRegionID = true;
+
+        public string Name
+        {
+            get { return "EtcdMonitoringModule"; }
+        }
+
+        public Type ReplaceableInterface
+        {
+            get { return null; }
+        }
+
+        public void Initialise(IConfigSource source)
+        {
+            if (source.Configs["Etcd"] == null)
+                return;
+
+            IConfig etcdConfig = source.Configs["Etcd"];
+
+            string etcdUrls = etcdConfig.GetString("EtcdUrls", String.Empty);
+            if (etcdUrls == String.Empty)
+                return;
+
+            m_etcdBasePath = etcdConfig.GetString("BasePath", m_etcdBasePath);
+            m_appendRegionID = etcdConfig.GetBoolean("AppendRegionID", m_appendRegionID);
+
+            if (!m_etcdBasePath.EndsWith("/"))
+                m_etcdBasePath += "/";
+
+            try
+            {
+                string[] endpoints = etcdUrls.Split(new char[] {','});
+                List<Uri> uris = new List<Uri>();
+                foreach (string endpoint in endpoints)
+                    uris.Add(new Uri(endpoint.Trim()));
+
+                m_client = new EtcdClient(uris.ToArray(), new DefaultSerializer(), new DefaultSerializer());
+            }
+            catch (Exception e)
+            {
+                m_log.DebugFormat("[ETCD]: Error initializing connection: " + e.ToString());
+                return;
+            }
+
+            m_log.DebugFormat("[ETCD]: Etcd module configured");
+            m_enabled = true;
+        }
+
+        public void Close()
+        {
+            //m_client = null;
+            m_scene = null;
+        }
+
+        public void AddRegion(Scene scene)
+        {
+            m_scene = scene;
+
+            if (m_enabled)
+            {
+                if (m_appendRegionID)
+                    m_etcdBasePath += m_scene.RegionInfo.RegionID.ToString() + "/";
+
+                m_log.DebugFormat("[ETCD]: Using base path {0} for all keys", m_etcdBasePath);
+
+                try
+                {
+                    m_client.Advanced.CreateDirectory(new CreateDirectoryRequest() {Key = m_etcdBasePath});
+                }
+                catch (Exception e)
+                {
+                    m_log.ErrorFormat("Exception trying to create base path {0}: " + e.ToString(), m_etcdBasePath);
+                }
+
+                scene.RegisterModuleInterface<IEtcdModule>(this);
+            }
+        }
+
+        public void RemoveRegion(Scene scene)
+        {
+        }
+
+        public void RegionLoaded(Scene scene)
+        {
+        }
+
+        public bool Store(string k, string v)
+        {
+            return Store(k, v, 0);
+        }
+
+        public bool Store(string k, string v, int ttl)
+        {
+            Response resp = m_client.Advanced.SetKey(new SetKeyRequest() { Key = m_etcdBasePath + k, Value = v, TimeToLive = ttl });
+
+            if (resp == null)
+                return false;
+
+            if (resp.ErrorCode.HasValue)
+            {
+                m_log.DebugFormat("[ETCD]: Error {0} ({1}) storing {2} => {3}", resp.Cause, (int)resp.ErrorCode, m_etcdBasePath + k, v);
+
+                return false;
+            }
+
+            return true;
+        }
+
+        public string Get(string k)
+        {
+            Response resp = m_client.Advanced.GetKey(new GetKeyRequest() { Key = m_etcdBasePath + k });
+
+            if (resp == null)
+                return String.Empty;
+
+            if (resp.ErrorCode.HasValue)
+            {
+                m_log.DebugFormat("[ETCD]: Error {0} ({1}) getting {2}", resp.Cause, (int)resp.ErrorCode, m_etcdBasePath + k);
+
+                return String.Empty;
+            }
+
+            return resp.Node.Value;
+        }
+
+        public void Delete(string k)
+        {
+            m_client.Advanced.DeleteKey(new DeleteKeyRequest() { Key = m_etcdBasePath + k });
+        }
+
+        public void Watch(string k, Action<string> callback)
+        {
+            m_client.Advanced.WatchKey(new WatchKeyRequest() { Key = m_etcdBasePath + k, Callback = (x) => { callback(x.Node.Value); } });
+        }
+    }
+}