GEEKING

Roblox.RccServiceArbiter/ArbiterInstaller.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+
+namespace Roblox.RccServiceArbiter
+{
+    [RunInstaller(true)]
+    public class ArbiterInstaller : Roblox.ServiceProcess.ServiceHostInstaller
+    {
+        public override string ServiceName
+        {
+            get { return "Roblox.RccServiceArbiter"; }
+        }
+
+        public override string DisplayName
+        {
+            get { return "Roblox RccService Arbiter"; }
+        }
+
+        public override string Description
+        {
+            get { return "Manages a handful of processes running the RCC web service."; }
+        }
+    }
+}

Roblox.RccServiceArbiter/IArbiter.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.ServiceModel;
+
+namespace Roblox.RccServiceArbiter
+{
+    [ServiceContract(Namespace = "http://watrbx.wtf/", Name = "Arbiter", ConfigurationName = "IArbiter")]
+    interface IArbiter
+    {
+        [OperationContract]
+        string GetStatsEx(bool clearExceptions);
+
+        [OperationContract]
+        string GetStats();
+
+        [OperationContract]
+        void SetMultiProcess(bool value);
+
+        [OperationContract]
+        void SetRecycleProcess(bool value);
+
+        [OperationContract]
+        void SetThreadConfigScript(string script, bool broadcast);
+
+        [OperationContract]
+        void SetRecycleQueueSize(int value);
+    };
+}

Roblox.RccServiceArbiter/IRccServiceArbiter.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Roblox.RccServiceArbiter
+{
+    [ServiceContract(ConfigurationName = "IRccServiceArbiter")]
+    interface IRccServiceArbiter
+    {
+    }
+}

Roblox.RccServiceArbiter/JobManager.cs

@@ -0,0 +1,949 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Diagnostics;
+using Roblox.Grid.Arbiter.Common;
+
+namespace Roblox.RccServiceArbiter
+{
+    class JobManager
+    {
+        private class RccServiceProcess
+        {
+            internal static string RCCServiceArgs = "/Console {0} {1} ";
+            internal static string RCCServiceCrashUploaderArgs = "/CrashReporter {0} {1} ";
+
+            private Process process;
+            //internal PublicDomain.Win32.Job job;77v7 
+            private int port;
+            private DateTime expirationTime;
+            public int numThreads = 1;
+
+            internal bool HasExited
+            {
+                get { return process.HasExited; }
+            }
+            internal DateTime ExpirationTime
+            {
+                get { return expirationTime; }
+                set { expirationTime = value; }
+            }
+
+            internal Roblox.Grid.Rcc.RCCServiceSoap SoapInterface
+            {
+                get
+                {
+                    Roblox.Grid.Rcc.RCCServiceSoap result = new Roblox.Grid.Rcc.RCCServiceSoap();
+                    result.Url = "http://localhost:" + port.ToString();
+                    result.Timeout = (int)Properties.Settings.Default.Timeout.TotalMilliseconds;
+                    return result;
+                }
+            }
+
+            internal RccServiceProcess(int port)
+            {
+                this.port = port;
+                this.process = new Process();
+            }
+
+            internal void Start(string exe)
+            {
+                process.StartInfo = new ProcessStartInfo(exe, String.Format(RCCServiceArgs, Properties.Settings.Default.RccServiceLaunch,port));
+                process.Start();
+            }
+            internal void StartCrashReporter(string exe)
+            {
+                process.StartInfo = new ProcessStartInfo(exe, String.Format(RCCServiceCrashUploaderArgs, Properties.Settings.Default.RccServiceLaunch, port));
+                process.Start();
+            }
+
+            internal void Close()
+            {
+                process.Kill();
+            }
+        }
+
+
+        private bool multiProcessMode;
+        private bool recycleProcessMode;
+        private int recycleJobCount;
+        private string startScript;
+        private string rccServiceExe;
+        private System.Threading.ReaderWriterLock exceptionLock;
+        private System.Collections.Generic.Dictionary<string, int> exceptionInformation;
+
+        private System.Threading.ReaderWriterLock activeJobsLock;
+
+        RccServiceProcess singleProcess;
+        private System.Collections.Generic.Dictionary<string, RccServiceProcess> activeJobs;
+        private System.Collections.Generic.List<string> lostJobs;
+        private System.Collections.Generic.List<string> closedJobs;
+
+        private System.Threading.Semaphore newProcessRequests;
+        private System.Threading.Semaphore newProcessReady;
+        private System.Threading.Thread newProcessThread;
+        private System.Threading.Thread expiredJobThread;
+        private System.Threading.Thread monitorThread;
+        private System.Threading.Thread clientSettingsThread;
+
+        private System.Collections.Generic.LinkedList<RccServiceProcess> recycledProcesses;
+        private System.Collections.Generic.LinkedList<RccServiceProcess> readyProcesses;
+
+        public enum ThreadPoolConfig { Threads1 = 1, Threads2 = 2, Threads3 = 3, Threads4 = 4, Threads8 = 8, Threads16 = 16, Auto = 101, PlayerCount = 102, JobCount = 103 }
+        private Dictionary<string, ThreadPoolConfig> threadPoolConfigList;
+        ThreadPoolConfig threadPoolConfig;
+
+        public JobManager()
+        {
+            System.Net.ServicePointManager.DefaultConnectionLimit = Properties.Settings.Default.MaxConnections;
+
+            this.multiProcessMode = !Properties.Settings.Default.SingleProcess;
+            this.recycleProcessMode = Properties.Settings.Default.RecycleProcesses;
+            this.recycleJobCount = 4;
+            this.startScript = Properties.Settings.Default.StartScript;
+
+            exceptionLock = new System.Threading.ReaderWriterLock();
+            exceptionInformation = new Dictionary<string, int>();
+            activeJobsLock = new System.Threading.ReaderWriterLock();
+            newProcessRequests = new System.Threading.Semaphore(0, 1000);
+            newProcessReady = new System.Threading.Semaphore(0, 1000);
+            singleProcess = null;
+
+            System.Reflection.Assembly assembly = System.Reflection.Assembly.GetExecutingAssembly();
+            string fullProcessPath = assembly.Location;
+            rccServiceExe = System.IO.Path.GetDirectoryName(fullProcessPath) + System.IO.Path.DirectorySeparatorChar + Properties.Settings.Default.RccServicePath;
+
+            activeJobs = new Dictionary<string, RccServiceProcess>();
+            lostJobs = new List<string>();
+            closedJobs = new List<string>();
+            recycledProcesses = new LinkedList<RccServiceProcess>();
+            readyProcesses = new LinkedList<RccServiceProcess>();
+
+            threadPoolConfig = ThreadPoolConfig.Threads1;
+
+            threadPoolConfigList = new Dictionary<string, ThreadPoolConfig>();
+            threadPoolConfigList.Add("Threads1", ThreadPoolConfig.Threads1);
+            threadPoolConfigList.Add("Threads2", ThreadPoolConfig.Threads2);
+            threadPoolConfigList.Add("Threads3", ThreadPoolConfig.Threads3);
+            threadPoolConfigList.Add("Threads4", ThreadPoolConfig.Threads4);
+            threadPoolConfigList.Add("Threads8", ThreadPoolConfig.Threads8);
+            threadPoolConfigList.Add("Threads16", ThreadPoolConfig.Threads16);
+            threadPoolConfigList.Add("PlayerCount", ThreadPoolConfig.PlayerCount);
+            threadPoolConfigList.Add("JobCount", ThreadPoolConfig.JobCount);
+        }
+        public void initialize()
+        {
+            Console.WriteLine("JobManager::initialize");
+
+            newProcessThread = new System.Threading.Thread(
+                delegate()
+                {
+                    while (true)
+                    {
+                        //Someone wants a job (for whatever reason)
+                        newProcessRequests.WaitOne();
+
+                        lock (recycledProcesses)
+                        {
+                            if(recycledProcesses.Count > 0){
+                                QueueProcess(recycledProcesses.First.Value);
+                                recycledProcesses.RemoveFirst();
+                                continue;
+                            }
+                        }
+
+                        Console.WriteLine("Processor Count" + Environment.ProcessorCount.ToString());
+                        //We need to make a new process
+                        QueueProcess(CreateNewProcess());
+                    }
+                });
+            newProcessThread.IsBackground = true; 
+            newProcessThread.Start();
+            newProcessRequests.Release(1);
+
+            expiredJobThread = new System.Threading.Thread(
+                delegate()
+                {
+                    while (true)
+                    {
+                        System.Threading.Thread.Sleep(60000);
+                        ClearExpiredJobs();
+                    }
+                });
+            expiredJobThread.IsBackground = true;
+            expiredJobThread.Start();
+
+            clientSettingsThread = new System.Threading.Thread(
+                delegate()
+                {
+                    while (true)
+                    {
+                        string configString = ClientSettings.FetchThreadPoolConfig();
+
+                        Console.WriteLine("FetchThreadPoolConfig got " + configString);
+
+                        if (configString != null)
+                        {
+                            ThreadPoolConfig config;
+                            if (threadPoolConfigList.TryGetValue(configString, out config))
+                            {
+                                SetTheadPoolConfig(config);
+                            }
+                            else
+                            {
+                                Console.WriteLine("Invalid ThreadPoolConfig");
+                            }
+                        }
+
+                        // check for new setting every 10 mins
+                        System.Threading.Thread.Sleep(10 * 60000);
+                    }
+                }
+                );
+
+            clientSettingsThread.IsBackground = true;
+            clientSettingsThread.Start();
+        }
+        ~JobManager()
+        {
+            foreach(RccServiceProcess process in readyProcesses)
+            {
+                CloseProcess(process);
+            }
+            foreach (RccServiceProcess process in activeJobs.Values)
+            {
+                CloseProcess(process);
+            }
+        }
+
+        public string CreateThreadConfigScript(int numThreads)
+        {
+            // we can have 1, 2, 3, 4, 8, 16 threads
+            int value = numThreads;
+            if (value > 4)
+            {
+                if (value <= 8)
+                    value = 8;
+                else
+                    value = 16;
+            }
+            return String.Format(@"settings()['Task Scheduler'].ThreadPoolConfig = Enum.ThreadPoolConfig.Threads{0};", value);
+        }
+
+        void StartPlayerCountMonitorThread()
+        {
+            if (monitorThread != null && monitorThread.IsAlive)
+                return;
+
+            // monitor player count and readjust task scheduler thread count
+            monitorThread = new System.Threading.Thread(
+                delegate()
+                {
+                    while (true)
+                    {
+                        // query player count
+                        try
+                        {
+                            activeJobsLock.AcquireReaderLock(-1);
+
+                            System.Collections.Generic.SynchronizedCollection<RccServiceProcess> set = new SynchronizedCollection<RccServiceProcess>();
+                            foreach (RccServiceProcess process in activeJobs.Values)
+                            {
+                                if (!set.Contains(process))
+                                {
+                                    int maxCount = 0;
+                                    using (var rccService = process.SoapInterface)
+                                    {    
+                                        Roblox.Grid.Rcc.Job[] jobs = rccService.GetAllJobs();
+                                        foreach (Roblox.Grid.Rcc.Job job in jobs)
+                                        {
+                                            Roblox.Grid.Rcc.ScriptExecution script = new Roblox.Grid.Rcc.ScriptExecution();
+                                            script.name = "GetPlayerCount";
+                                            script.script = "return #game.Players:GetChildren()";
+                                            Roblox.Grid.Rcc.LuaValue[] result = rccService.Execute(job.id, script);
+                                            int playerCount = Int32.Parse(result[0].value);
+                                            if (maxCount < playerCount)
+                                                maxCount = playerCount;
+
+                                            Console.WriteLine(String.Format("Job {0}, player count: {1}", job.id, playerCount));
+                                        }
+                                    }
+
+                                    // set thread count based on number of players in game
+                                    ConfigureProcess(process, "ConfigThread", CreateThreadConfigScript((maxCount / 100 * Environment.ProcessorCount) + 1));
+                                    
+                                    set.Add(process);
+                                }
+                            }
+                        }
+                        finally
+                        {
+                            activeJobsLock.ReleaseReaderLock();
+                        }
+
+                        // do this every 5 mins
+                        System.Threading.Thread.Sleep(5 * 60000);
+
+                    }
+                }
+                );
+
+            monitorThread.IsBackground = true;
+            monitorThread.Start();
+        }
+
+        void StopPlayerCountMonitorThread()
+        {
+            if (monitorThread != null)
+                monitorThread.Abort();
+
+            monitorThread = null;
+        }
+
+        public void SetTheadPoolConfig(ThreadPoolConfig config)
+        {
+            if (config == threadPoolConfig)
+                return;
+
+            Console.WriteLine("SetThreadPoolConfig " + config.ToString());
+
+            if (threadPoolConfig == ThreadPoolConfig.PlayerCount)
+                StopPlayerCountMonitorThread();
+
+            switch (config)
+            {
+                case ThreadPoolConfig.JobCount:
+                    AdjustThreadsPerProcessByJobCount();
+                    break;
+                case ThreadPoolConfig.PlayerCount:
+                    StartPlayerCountMonitorThread();
+                    break;
+                case ThreadPoolConfig.Auto:
+                    SetThreadsPerProcess(Environment.ProcessorCount, true);
+                    break;
+                default:
+                    SetThreadsPerProcess((int)config, true);
+                    break;
+
+            }
+
+            threadPoolConfig = config;
+        }
+
+        public void SetMultiProcess(bool value)
+        {
+            multiProcessMode = value;
+        }
+
+        public void SetRecycleProcess(bool value)
+        {
+            recycleProcessMode = value;
+        }
+
+        public void SetRecycleProcessCount(int count)
+        {
+            recycleJobCount = count;
+        }
+
+        public void SetStartScript(string value, bool broadcast)
+        {
+            startScript = value;
+            if (broadcast)
+            {
+                try{
+                    activeJobsLock.AcquireReaderLock(-1);
+
+                    System.Collections.Generic.SynchronizedCollection<RccServiceProcess> set = new SynchronizedCollection<RccServiceProcess>();
+                    foreach (RccServiceProcess process in activeJobs.Values)
+                    {
+                        if (!set.Contains(process))
+                        {
+                            ConfigureProcess(process, "SetupThreads", startScript);
+                            set.Add(process);
+                        }
+                    }
+                }
+                finally
+                { 
+                    activeJobsLock.ReleaseReaderLock(); 
+                }
+            }
+        }
+
+        public void SetThreadsPerProcess(int value, bool broadcast)
+        {
+            Console.WriteLine("SetThreadPerProcess");
+
+            if (broadcast)
+            {
+                try
+                {
+                    activeJobsLock.AcquireReaderLock(-1);
+
+                    string script = CreateThreadConfigScript(value);
+
+                    Console.WriteLine("Created " + script);
+                    
+                    System.Collections.Generic.SynchronizedCollection<RccServiceProcess> set = new SynchronizedCollection<RccServiceProcess>();
+                    foreach (RccServiceProcess process in activeJobs.Values)
+                    {
+                        if (!set.Contains(process) && process.numThreads != value)
+                        {
+                            ConfigureProcess(process, "SetupThreads", script);
+                            process.numThreads = value; 
+                            set.Add(process);
+                        }
+                    }
+                }
+                finally
+                {
+                    activeJobsLock.ReleaseReaderLock();
+                }
+            }
+        }
+
+        public void AdjustThreadsPerProcessByJobCount()
+        {
+            int count = activeJobs.Count;
+            int numThreads = Environment.ProcessorCount;
+            if ((count > 1))
+            {
+                if (count % 2 == 1)
+                    count++;
+
+                numThreads = Environment.ProcessorCount / count;
+                if (numThreads < 1)
+                    numThreads = 1;
+            }
+
+            SetThreadsPerProcess(numThreads, true);
+        }
+
+        public string GetStats(bool clearExceptions)
+        {
+            ArbiterStats stats = new ArbiterStats();
+
+            activeJobsLock.AcquireReaderLock(-1);
+            try
+            {
+                stats.AddStat("Active Jobs", activeJobs.Count);
+            }
+            finally
+            {
+                activeJobsLock.ReleaseReaderLock();
+            }
+
+            stats.AddStat("Pending Jobs", PendingProcessCount);
+            stats.AddStat("Recycled Jobs", RecycleProcessCount);
+            System.Diagnostics.Process[] processlist = System.Diagnostics.Process.GetProcesses();
+            {
+                int RccServiceCount = 0;
+                long totalWorkingSet = 0;
+                long totalVirtual = 0;
+                long totalPrivateMemory = 0;
+                TimeSpan totalProcessorTime = new TimeSpan();
+                foreach (System.Diagnostics.Process theprocess in processlist)
+                {
+                    if (theprocess.ProcessName == "RCCService")
+                    {
+                        RccServiceCount++;
+
+                        totalWorkingSet += theprocess.WorkingSet64;
+                        totalVirtual += theprocess.VirtualMemorySize64;
+                        totalProcessorTime += theprocess.TotalProcessorTime;
+                        totalPrivateMemory += theprocess.PrivateMemorySize64;
+                    }
+                }
+                stats.AddStat("RccService Count", RccServiceCount);
+                stats.AddStat("Total WorkingSet", totalWorkingSet);
+                stats.AddStat("Total VirtualMemory", totalVirtual);
+                stats.AddStat("Total PrivateMemory", totalPrivateMemory);
+                stats.AddStat("Total ProcessorTime", totalProcessorTime.TotalSeconds);
+            }
+
+            stats.AddStat("Setting-MultipleProcess", multiProcessMode);
+            stats.AddStat("Setting-RecycleProcess", recycleProcessMode);
+            stats.AddStat("Setting-RecycleQueueSize", recycleJobCount);
+            stats.AddStat("Setting-RccServicePath", Properties.Settings.Default.RccServicePath);
+            
+            if(clearExceptions)
+                exceptionLock.AcquireWriterLock(-1);
+            else
+                exceptionLock.AcquireReaderLock(-1);
+            try
+            {
+                foreach (string message in exceptionInformation.Keys)
+                {
+                    stats.AddStat("Exception-" + message, exceptionInformation[message]);
+                }
+
+                if (clearExceptions)
+                {
+                    exceptionInformation.Clear();
+                }
+            }
+            finally
+            {
+                if (clearExceptions)
+                    exceptionLock.ReleaseWriterLock();
+                else
+                    exceptionLock.ReleaseReaderLock();
+            }
+
+            return stats.ToXml();
+        }
+
+        private static int GetPort()
+        {
+            return TcpPort.FindNextAvailablePort(64000);
+        }
+        private RccServiceProcess GetSingleProcess()
+        {
+            lock(readyProcesses)
+            {
+                if(singleProcess != null && !singleProcess.HasExited)
+                {
+                    return singleProcess;
+                }
+                else{
+                    singleProcess = null;
+                }
+            }
+
+            while (true)
+            {
+                bool requestNewJob = false;
+                if (PendingProcessCount == 0)
+                    requestNewJob = true;
+
+                if (requestNewJob)
+                {
+                    newProcessRequests.Release(1);
+                    newProcessReady.WaitOne();
+                }
+
+                lock (readyProcesses)
+                {
+                    if (singleProcess != null && !singleProcess.HasExited)
+                        return singleProcess;
+
+                    Debug.Assert(readyProcesses.Count > 0);
+                    singleProcess = readyProcesses.First.Value;
+                    readyProcesses.RemoveFirst();
+
+                    if (!singleProcess.HasExited)
+                    {
+                        return singleProcess;
+                    }
+                }
+            }
+        }
+
+        private RccServiceProcess GetNewProcess(double expirationInSeconds)
+        {
+            if (!multiProcessMode)
+            {
+                return GetSingleProcess();
+            }
+            else
+            {
+                while (true)
+                {
+                    newProcessRequests.Release(1);
+                    newProcessReady.WaitOne();
+                    
+                    RccServiceProcess result;
+                    lock (readyProcesses)
+                    {
+                        Debug.Assert(readyProcesses.Count > 0);
+                        result = readyProcesses.First.Value;
+                        readyProcesses.RemoveFirst();
+                    }
+
+                    if (result.HasExited)
+                    {
+                        //The pending job died, so try again
+                        continue;
+                    }
+
+                    result.ExpirationTime = DateTime.Now.AddSeconds(expirationInSeconds);
+                    return result;
+                }
+            }
+        }
+        private void RecycleProcess(RccServiceProcess process)
+        {
+            lock (recycledProcesses)
+            {
+                recycledProcesses.AddLast(process);
+            }
+        }
+        private void QueueProcess(RccServiceProcess process)
+        {
+            lock (readyProcesses)
+            {
+                readyProcesses.AddLast(process);
+            }
+            newProcessReady.Release(1);
+        }
+        private int RecycleProcessCount
+        {
+            get
+            {
+                lock (recycledProcesses)
+                {
+                    return recycledProcesses.Count;
+                }
+            }
+        }
+        private int PendingProcessCount
+        {
+            get
+            {
+                lock (readyProcesses)
+                {
+                    return readyProcesses.Count;
+                }
+            }
+        }
+        private string RecordException(string message)
+        {
+            try
+            {
+                exceptionLock.AcquireWriterLock(-1);
+                if (!exceptionInformation.ContainsKey(message))
+                {
+                    exceptionInformation[message] = 0;
+                }
+                exceptionInformation[message]++;
+            }
+            finally
+            {
+                exceptionLock.ReleaseWriterLock();
+            }
+            return message;
+        }
+
+        private bool ConfigureProcess(RccServiceProcess process, string scriptName, string script)
+        {
+            try
+            {
+                using (var rccService = process.SoapInterface)
+                {
+                    Roblox.Grid.Rcc.Job configJob = new Roblox.Grid.Rcc.Job();
+                    configJob.id = "Config";
+                    configJob.category = 0;
+                    configJob.cores = 1;
+                    configJob.expirationInSeconds = 10;
+                    Roblox.Grid.Rcc.ScriptExecution scriptExcution = new Roblox.Grid.Rcc.ScriptExecution();
+                    scriptExcution.name = scriptName;
+                    scriptExcution.script = script;
+                    scriptExcution.arguments = null;
+
+                    Console.WriteLine("ConfigureProcess, script: " + script);
+
+                    rccService.OpenJob(configJob, scriptExcution);
+                    rccService.CloseJob(configJob.id);
+                    return true;
+                }
+            }
+            catch (Exception)
+            {
+                RecordException("SetupScriptFailed");
+                return false;
+            }
+        }
+        private RccServiceProcess CreateCrashUploaderProcess()
+        {
+            RccServiceProcess result = new RccServiceProcess(GetPort());
+            result.StartCrashReporter(rccServiceExe);
+            return result;
+        }
+
+        private RccServiceProcess CreateNewProcess()
+        {
+            Console.WriteLine("CreateNewProcess");
+            RccServiceProcess result = new RccServiceProcess(GetPort());
+            result.Start(rccServiceExe);
+
+            if (ConfigureProcess(result, "SetupThreads", startScript))
+            {
+                return result;
+            }
+            else
+            {
+                //return it non-configured
+                return result;
+            }
+
+        }
+        private void ClearExpiredJobs()
+        {
+            List<string> jobsToRemove = new List<string>();
+            {
+                activeJobsLock.AcquireReaderLock(-1);
+                try
+                {
+                    DateTime currentTime = DateTime.Now;
+                    foreach (string jobId in activeJobs.Keys)
+                    {
+                        if (activeJobs[jobId].HasExited || activeJobs[jobId].ExpirationTime < currentTime)
+                        {
+                            jobsToRemove.Add(jobId);
+                        }
+                    }
+                }
+                finally
+                {
+                    activeJobsLock.ReleaseReaderLock();
+                }
+            }
+            foreach (string jobId in jobsToRemove)
+            {
+                RecordException("JobLeaseExpired");
+                CloseJob(jobId);
+            }
+        }
+        public void RenewLease(string jobId, double expirationInSeconds)
+        {
+            activeJobsLock.AcquireWriterLock(-1);
+            try
+            {
+                if (activeJobs.ContainsKey(jobId))
+                {
+                    if (!activeJobs[jobId].HasExited)
+                    {
+                        DateTime newTime = DateTime.Now.AddSeconds(expirationInSeconds);
+                        if (activeJobs[jobId].ExpirationTime < newTime)
+                        {
+                            activeJobs[jobId].ExpirationTime = newTime;
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                activeJobsLock.ReleaseWriterLock();
+            }
+        }
+        public Roblox.Grid.Rcc.RCCServiceSoap NewJob(string jobId, double expirationInSeconds, string task)
+        {
+            RccServiceProcess process = GetNewProcess(expirationInSeconds);
+
+            activeJobsLock.AcquireWriterLock(-1);
+            try
+            {
+                //Launch a process on a new port
+                activeJobs[jobId] = process;
+            }
+            finally
+            {
+                activeJobsLock.ReleaseWriterLock();
+            }
+
+            Console.WriteLine("NewJob, total: " + activeJobs.Count.ToString());
+
+            if (threadPoolConfig == ThreadPoolConfig.JobCount)
+                AdjustThreadsPerProcessByJobCount();
+
+            return GetJob(jobId, task, "");
+        }
+        public Roblox.Grid.Rcc.RCCServiceSoap GetJob(string jobId, string task, string extraInfo)
+        {
+            string message ="UnknownGetJobError";
+            activeJobsLock.AcquireReaderLock(-1);
+            try
+            {
+                if (activeJobs.ContainsKey(jobId))
+                {
+                    if (!activeJobs[jobId].HasExited)
+                    {
+                        return activeJobs[jobId].SoapInterface;
+                    }
+                    else
+                    {
+                        message = RecordException("JobDied");
+                        //A crash occurred, we need to spawn a process to deal with this.
+                        CreateCrashUploaderProcess();
+                        
+                        System.Threading.LockCookie cookie = activeJobsLock.UpgradeToWriterLock(-1);
+                        try
+                        {
+
+                            if (activeJobs.ContainsKey(jobId))
+                            {
+                                activeJobs.Remove(jobId);
+                            }
+                        }
+                        finally
+                        {
+                            activeJobsLock.DowngradeFromWriterLock(ref cookie);
+                        }
+                    }
+                }
+                else
+                {
+                    if (extraInfo != "")
+                    {
+                        message = RecordException("JobLost('" + extraInfo + "')");
+                    }
+                    if (!lostJobs.Contains(jobId))
+                    {
+                        lostJobs.Add(jobId);
+                        if (lostJobs.Count > 100)
+                        {
+                            lostJobs.RemoveAt(0);
+                        }
+
+                        if (closedJobs.Contains(jobId))
+                        {
+                            message = RecordException("JobLost-Unique" + task + "AfterClose");
+                        }
+                        else
+                        {
+                            message = RecordException("JobLost-Unique" + task);
+                        }
+                    }
+                    else
+                    {
+                        if (closedJobs.Contains(jobId))
+                        {
+                            message = RecordException("JobLost-" + task + "AfterClose");
+                        }
+                        else
+                        {
+                            message = RecordException("JobLost-" + task);
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                activeJobsLock.ReleaseReaderLock();
+            }
+            throw new Exception(message);
+        }
+        public Roblox.Grid.Rcc.RCCServiceSoap AnyJob(double expirationInSeconds)
+        {
+            activeJobsLock.AcquireReaderLock(-1);
+            try
+            {
+                foreach (RccServiceProcess p in activeJobs.Values)
+                {
+                    if (!p.HasExited)
+                    {
+                        return p.SoapInterface;
+                    }
+                }
+            }
+            finally
+            {
+                activeJobsLock.ReleaseReaderLock();
+            }
+
+            //We have no jobs, grab a new one
+            RccServiceProcess newProcess = GetNewProcess(expirationInSeconds);
+
+            //Queue it back up since we won't be using it for long
+            RecycleProcess(newProcess);
+
+            return newProcess.SoapInterface;
+        }
+        private void CloseProcess(RccServiceProcess process)
+        {
+            if (!process.HasExited)
+            {
+                process.Close();
+            }
+        }
+
+        public void CloseJob(string jobId)
+        {
+            activeJobsLock.AcquireWriterLock(-1);
+            try
+            {
+                if (activeJobs.ContainsKey(jobId))
+                {
+                    
+                    if (!multiProcessMode)
+                    {
+                        if (activeJobs[jobId] != singleProcess)
+                        {
+                            //Our process is a hanger on, not the core singleProcess
+                            //go ahead and close it down
+                            CloseProcess(activeJobs[jobId]);
+                        }
+                    }
+                    else if (recycleProcessMode)
+                    {
+                        //Requeue it for use later, unless its exited already
+                        if (!activeJobs[jobId].HasExited)
+                        {
+                            if (RecycleProcessCount < recycleJobCount)
+                            {
+                                RecycleProcess(activeJobs[jobId]);
+                            }
+                            else
+                            {
+                                //Close it, we probably have too many jobs already
+                                CloseProcess(activeJobs[jobId]);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        //Close it, we're creating a new process for every job
+                        CloseProcess(activeJobs[jobId]);
+                    }
+
+                    //Kick it out of the running job list
+                    activeJobs.Remove(jobId);
+
+                    closedJobs.Add(jobId);
+                    if (closedJobs.Count > 100)
+                    {
+                        closedJobs.RemoveAt(0);
+                    }
+
+                    Console.WriteLine("CloseJob, total: " + activeJobs.Count.ToString());
+
+                    if (threadPoolConfig == ThreadPoolConfig.JobCount)
+                        AdjustThreadsPerProcessByJobCount();
+
+                }
+                else
+                {
+                    RecordException("JobLost-Close");
+                    System.Console.Out.WriteLine("Job[" + jobId + "] was lost");
+                }
+            }
+            finally
+            {
+                activeJobsLock.ReleaseWriterLock();
+            }
+        }
+
+        public delegate void JobDelegate(Roblox.Grid.Rcc.RCCServiceSoap soapProcess);
+        public void DispatchRequest(JobDelegate jobDelegate)
+        {
+            activeJobsLock.AcquireReaderLock(-1);
+            try
+            {
+                foreach (RccServiceProcess process in activeJobs.Values)
+                {
+                    if (!process.HasExited)
+                    {
+                        using (var rccSoap = process.SoapInterface)
+                        {
+                            jobDelegate(rccSoap);
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                activeJobsLock.ReleaseReaderLock();
+            }
+        }
+    }
+}

Roblox.RccServiceArbiter/Log.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+namespace Roblox.RccServiceArbiter
+{
+    internal static class Log
+    {
+        static StreamWriter log;
+        static Log()
+        {
+            if (Properties.Settings.Default.LogTransactions)
+            {
+                log = new StreamWriter(String.Format("C:\\arbiter-log{0}.txt", Guid.NewGuid().ToString()));
+            }
+        }
+
+        static internal void Event(string message)
+        {
+            if (log != null)
+            {
+                lock (log)
+                {
+                    log.WriteLine(String.Format("{0} - {1}", DateTime.Now.ToString(), message));
+                    log.Flush();
+                }
+            }
+        }
+
+    }
+}

Roblox.RccServiceArbiter/Program.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Roblox.RccServiceArbiter
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Log.Event("Starting");
+            //Cleanup old RCCService instances
+            System.Diagnostics.Process[] processlist = System.Diagnostics.Process.GetProcesses();
+            foreach(System.Diagnostics.Process theprocess in processlist)
+            {
+                if (theprocess.ProcessName == "RCCService")
+                {
+                    theprocess.Kill();
+                }
+            }
+
+            RccService rccService = new RccService();
+            RccServiceMonitor rccServiceMonitor = new RccServiceMonitor(rccService);
+            Roblox.ServiceProcess.ServiceBasePublic[] otherHosts = new Roblox.ServiceProcess.ServiceBasePublic[1];
+            otherHosts[0] = new Roblox.ServiceProcess.ServiceHostApp<RccServiceMonitor>(rccServiceMonitor);
+            var app = new Roblox.ServiceProcess.ServiceHostApp<RccService>(rccService, otherHosts);
+            app.HostOpened += new EventHandler(delegate(object a,EventArgs b) { rccService.initializeJobManager(); });
+            //var app = new Roblox.ServiceProcess.ServiceHostApp<RccServiceMonitor>(rccServiceMonitor);
+            app.Process(args, delegate() { Roblox.Grid.Arbiter.Common.ArbiterStats stats = new Roblox.Grid.Arbiter.Common.ArbiterStats(rccService.GetStats(false));
+                                           Console.Out.Write(stats.ToString());
+            });
+        }
+    }
+}

Roblox.RccServiceArbiter/Properties/AssemblyInfo.cs

@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Roblox.RccServiceArbiter")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Roblox.RccServiceArbiter")]
+[assembly: AssemblyCopyright("Copyright ©  2010")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("c772546d-c064-463b-a4ef-acf7c020d2cb")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

Roblox.RccServiceArbiter/Properties/Settings.Designer.cs

@@ -0,0 +1,148 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+namespace Roblox.RccServiceArbiter.Properties {
+    
+    
+    [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
+    [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "16.10.0.0")]
+    internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase {
+        
+        private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
+        
+        public static Settings Default {
+            get {
+                return defaultInstance;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("False")]
+        public bool RecycleProcesses {
+            get {
+                return ((bool)(this["RecycleProcesses"]));
+            }
+            set {
+                this["RecycleProcesses"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("False")]
+        public bool SingleProcess {
+            get {
+                return ((bool)(this["SingleProcess"]));
+            }
+            set {
+                this["SingleProcess"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("False")]
+        public bool Verbose {
+            get {
+                return ((bool)(this["Verbose"]));
+            }
+            set {
+                this["Verbose"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("..\\RCCService.exe")]
+        public string RccServicePath {
+            get {
+                return ((string)(this["RccServicePath"]));
+            }
+            set {
+                this["RccServicePath"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("/content:content\\\\")]
+        public string RccServiceLaunch {
+            get {
+                return ((string)(this["RccServiceLaunch"]));
+            }
+            set {
+                this["RccServiceLaunch"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("False")]
+        public bool LogTransactions {
+            get {
+                return ((bool)(this["LogTransactions"]));
+            }
+            set {
+                this["LogTransactions"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("00:05:00")]
+        public global::System.TimeSpan Timeout {
+            get {
+                return ((global::System.TimeSpan)(this["Timeout"]));
+            }
+            set {
+                this["Timeout"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("128")]
+        public int MaxConnections {
+            get {
+                return ((int)(this["MaxConnections"]));
+            }
+            set {
+                this["MaxConnections"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("\r\n          settings()[\'Task Scheduler\'].ThreadPoolConfig = Enum.ThreadPoolConfig" +
+            ".Threads1;\r\n          -- deprecated settings()[\'Task Scheduler\']:SetThreadShare(" +
+            "1000,4);\r\n        ")]
+        public string StartScript {
+            get {
+                return ((string)(this["StartScript"]));
+            }
+            set {
+                this["StartScript"] = value;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("..\\AppSettings.xml")]
+        public string AppSettingPath {
+            get {
+                return ((string)(this["AppSettingPath"]));
+            }
+            set {
+                this["AppSettingPath"] = value;
+            }
+        }
+    }
+}

Roblox.RccServiceArbiter/Properties/Settings.settings

@@ -0,0 +1,39 @@
+<?xml version='1.0' encoding='utf-8'?>
+<SettingsFile xmlns="http://schemas.microsoft.com/VisualStudio/2004/01/settings" CurrentProfile="(Default)" GeneratedClassNamespace="Roblox.RccServiceArbiter.Properties" GeneratedClassName="Settings">
+  <Profiles />
+  <Settings>
+    <Setting Name="RecycleProcesses" Type="System.Boolean" Scope="User">
+      <Value Profile="(Default)">False</Value>
+    </Setting>
+    <Setting Name="SingleProcess" Type="System.Boolean" Scope="User">
+      <Value Profile="(Default)">False</Value>
+    </Setting>
+    <Setting Name="Verbose" Type="System.Boolean" Scope="User">
+      <Value Profile="(Default)">False</Value>
+    </Setting>
+    <Setting Name="RccServicePath" Type="System.String" Scope="User">
+      <Value Profile="(Default)">..\RCCService.exe</Value>
+    </Setting>
+    <Setting Name="RccServiceLaunch" Type="System.String" Scope="User">
+      <Value Profile="(Default)">/content:content\\</Value>
+    </Setting>
+    <Setting Name="LogTransactions" Type="System.Boolean" Scope="User">
+      <Value Profile="(Default)">False</Value>
+    </Setting>
+    <Setting Name="Timeout" Type="System.TimeSpan" Scope="User">
+      <Value Profile="(Default)">00:05:00</Value>
+    </Setting>
+    <Setting Name="MaxConnections" Type="System.Int32" Scope="User">
+      <Value Profile="(Default)">128</Value>
+    </Setting>
+    <Setting Name="StartScript" Type="System.String" Scope="User">
+      <Value Profile="(Default)">
+          settings()['Task Scheduler'].ThreadPoolConfig = Enum.ThreadPoolConfig.Threads1;
+          -- deprecated settings()['Task Scheduler']:SetThreadShare(1000,4);
+        </Value>
+    </Setting>
+    <Setting Name="AppSettingPath" Type="System.String" Scope="User">
+      <Value Profile="(Default)">..\AppSettings.xml</Value>
+    </Setting>
+  </Settings>
+</SettingsFile>

Roblox.RccServiceArbiter/RccService.cs

@@ -0,0 +1,521 @@
+using System;
+using System.Collections.Generic;
+using System.ServiceModel;
+using System.Text;
+using System.Diagnostics;
+
+namespace Roblox.RccServiceArbiter
+{
+    [ServiceBehavior(InstanceContextMode = InstanceContextMode.Single, ConcurrencyMode = ConcurrencyMode.Multiple)]
+    [System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")]
+    [System.ServiceModel.ServiceContractAttribute(Namespace = "http://watrbx.wtf/", ConfigurationName = "RCCServiceSoap")]
+    [System.ServiceModel.XmlSerializerFormat(Style=OperationFormatStyle.Document, Use=OperationFormatUse.Literal)]
+    class RccService : Roblox.Grid.Rcc.Server.RCCServiceSoap
+    {
+        private JobManager jobManager;
+
+        public RccService()
+        {
+            jobManager = new JobManager();
+        }
+        public void initializeJobManager()
+        {
+            jobManager.initialize();
+        }
+        #region Converters
+
+
+        private static Roblox.Grid.Rcc.Job convert(Roblox.Grid.Rcc.Server.Job job)
+        {
+            Roblox.Grid.Rcc.Job result = new Roblox.Grid.Rcc.Job();
+            result.id = job.id;
+            result.category = job.category;
+            result.cores = job.cores;
+            result.expirationInSeconds = job.expirationInSeconds;
+            return result;
+        }
+        private static Roblox.Grid.Rcc.Server.Job convert(Roblox.Grid.Rcc.Job job)
+        {
+            Roblox.Grid.Rcc.Server.Job result = new Roblox.Grid.Rcc.Server.Job();
+            result.id = job.id;
+            result.category = job.category;
+            result.cores = job.cores;
+            result.expirationInSeconds = job.expirationInSeconds;
+            return result;
+        }
+        private static Roblox.Grid.Rcc.LuaType convert(Roblox.Grid.Rcc.Server.LuaType luaType)
+        {
+            switch (luaType)
+            {
+                case Roblox.Grid.Rcc.Server.LuaType.LUA_TBOOLEAN: return Roblox.Grid.Rcc.LuaType.LUA_TBOOLEAN;
+                case Roblox.Grid.Rcc.Server.LuaType.LUA_TNIL: return Roblox.Grid.Rcc.LuaType.LUA_TNIL;
+                case Roblox.Grid.Rcc.Server.LuaType.LUA_TNUMBER: return Roblox.Grid.Rcc.LuaType.LUA_TNUMBER;
+                case Roblox.Grid.Rcc.Server.LuaType.LUA_TSTRING: return Roblox.Grid.Rcc.LuaType.LUA_TSTRING;
+                case Roblox.Grid.Rcc.Server.LuaType.LUA_TTABLE: return Roblox.Grid.Rcc.LuaType.LUA_TTABLE;
+            }
+            throw new Exception("Unknown LuaType");
+        }
+        private static Roblox.Grid.Rcc.Server.LuaType convert(Roblox.Grid.Rcc.LuaType luaType)
+        {
+            switch (luaType)
+            {
+                case Roblox.Grid.Rcc.LuaType.LUA_TBOOLEAN: return Roblox.Grid.Rcc.Server.LuaType.LUA_TBOOLEAN;
+                case Roblox.Grid.Rcc.LuaType.LUA_TNIL: return Roblox.Grid.Rcc.Server.LuaType.LUA_TNIL;
+                case Roblox.Grid.Rcc.LuaType.LUA_TNUMBER: return Roblox.Grid.Rcc.Server.LuaType.LUA_TNUMBER;
+                case Roblox.Grid.Rcc.LuaType.LUA_TSTRING: return Roblox.Grid.Rcc.Server.LuaType.LUA_TSTRING;
+                case Roblox.Grid.Rcc.LuaType.LUA_TTABLE: return Roblox.Grid.Rcc.Server.LuaType.LUA_TTABLE;
+            }
+            throw new Exception("Unknown LuaType");
+        }
+        private static Roblox.Grid.Rcc.LuaValue[] convert(Roblox.Grid.Rcc.Server.LuaValue[] luaValues)
+        {
+            if (luaValues == null) return null;
+
+            Roblox.Grid.Rcc.LuaValue[] result = new Roblox.Grid.Rcc.LuaValue[luaValues.Length];
+            int i = 0;
+            foreach (Roblox.Grid.Rcc.Server.LuaValue luaValue in luaValues)
+            {
+                result[i++] = convert(luaValue);
+            }
+            return result;
+        }
+        private static Roblox.Grid.Rcc.Server.LuaValue[] convert(Roblox.Grid.Rcc.LuaValue[] luaValues)
+        {
+            if (luaValues == null) return null;
+
+            Roblox.Grid.Rcc.Server.LuaValue[] result = new Roblox.Grid.Rcc.Server.LuaValue[luaValues.Length];
+            int i = 0;
+            foreach (Roblox.Grid.Rcc.LuaValue luaValue in luaValues)
+            {
+                result[i++] = convert(luaValue);
+            }
+            return result;
+        }
+
+
+        private static Roblox.Grid.Rcc.LuaValue convert(Roblox.Grid.Rcc.Server.LuaValue luaValue)
+        {
+            Roblox.Grid.Rcc.LuaValue result = new Roblox.Grid.Rcc.LuaValue();
+            result.type = convert(luaValue.type);
+            result.value = luaValue.value;
+            result.table = convert(luaValue.table);
+            return result;
+        }
+
+        private static Roblox.Grid.Rcc.Server.LuaValue convert(Roblox.Grid.Rcc.LuaValue luaValue)
+        {
+            Roblox.Grid.Rcc.Server.LuaValue result = new Roblox.Grid.Rcc.Server.LuaValue();
+            result.type = convert(luaValue.type);
+            result.value = luaValue.value;
+            result.table = convert(luaValue.table);
+            return result;
+        }
+
+
+        private static Roblox.Grid.Rcc.ScriptExecution convert(Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            Roblox.Grid.Rcc.ScriptExecution result = new Roblox.Grid.Rcc.ScriptExecution();
+            result.name = script.name;
+            result.script = script.script;
+            result.arguments = convert(script.arguments);
+            return result;
+        }
+
+        #endregion 
+
+        private string ToString(Roblox.Grid.Rcc.Server.LuaValue[] values)
+        {
+            string result = "";
+            foreach (Roblox.Grid.Rcc.Server.LuaValue value in values)
+            {
+                switch (value.type)
+                {
+                    case Roblox.Grid.Rcc.Server.LuaType.LUA_TBOOLEAN:
+                    case Roblox.Grid.Rcc.Server.LuaType.LUA_TNUMBER:
+                    case Roblox.Grid.Rcc.Server.LuaType.LUA_TSTRING:
+                        result += value.value + "\n";
+                        break;
+                    case Roblox.Grid.Rcc.Server.LuaType.LUA_TNIL:
+                        result += "[NIL]";
+                        break;
+                    case Roblox.Grid.Rcc.Server.LuaType.LUA_TTABLE:
+                        result += "{\n" + ToString(value.table) + "}\n";
+                        break;
+                }
+            }
+            return result;
+        }
+        public string GetStats(bool clearExceptions)
+        {
+            return jobManager.GetStats(clearExceptions);
+        }
+
+        public void SetMultiProcess(bool value)
+        {
+            jobManager.SetMultiProcess(value);
+        }
+
+        public void SetRecycleProcess(bool value)
+        {
+            jobManager.SetRecycleProcess(value);
+        }
+        public void SetRecycleProcessCount(int value)
+        {
+            jobManager.SetRecycleProcessCount(value);
+        }
+
+
+
+        public void SetThreadConfigScript(string script, bool broadcast)
+        {
+            jobManager.SetStartScript(script, broadcast);
+        }
+
+        #region RCCServiceSoap Members
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/HelloWorld", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override string HelloWorld()
+        {
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("HelloWorld()");
+
+            try
+            {
+                string result;
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.AnyJob(30))
+                {
+                    result = rccService.HelloWorld();
+                }
+                Log.Event("HelloWorld - Success");
+                return result;
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("HelloWorld - Exception - {0}", e.Message));
+                throw;
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/GetVersion", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override string GetVersion()
+        {
+            //if (Properties.Settings.Default.Verbose) Console.WriteLine("GetVersion()");
+            using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.AnyJob(30))
+            {
+                return rccService.GetVersion();
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/OpenJob", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormat(Style = OperationFormatStyle.Document, Use = OperationFormatUse.Literal)]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] OpenJob(Roblox.Grid.Rcc.Server.Job job, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            System.Threading.ThreadPool.QueueUserWorkItem((dummy) =>
+            {
+                try
+                {
+                    OpenJobEx(job, script); 
+                }
+                catch (Exception ex)
+                {
+                    ExceptionHandler.LogException(ex);
+                }
+            });
+            return null; 
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/OpenJobEx", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormat(Style = OperationFormatStyle.Document, Use = OperationFormatUse.Literal)]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] OpenJobEx(Roblox.Grid.Rcc.Server.Job job, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            try
+            {
+                if (Properties.Settings.Default.Verbose) Console.WriteLine("OpenJob(" + job.ToString() + ")");
+
+                Console.WriteLine("OpenJobEx, script name: " + script.name);
+                Roblox.Grid.Rcc.Server.LuaValue[] finalResult;
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.NewJob(job.id, job.expirationInSeconds, "OpenJobEx"))
+                {
+                    //Create a new process, and have it execute this Job
+                    Roblox.Grid.Rcc.LuaValue[] result = rccService.OpenJob(convert(job), convert(script));
+                    finalResult = convert(result);
+                }
+
+                Log.Event(String.Format("OpenJobEx('{0}', {1}, {2}) - Success", job.id, job.category, job.expirationInSeconds));
+                return finalResult;
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("OpenJobEx('{0}', {1}, {2}) - Exception - {3}", job.id, job.category, job.expirationInSeconds, e.Message));
+                throw;
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/BatchJob", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] BatchJob(Roblox.Grid.Rcc.Server.Job job, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            throw new Exception("BatchJob is deprecated, use BatchJobEx");
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/BatchJobEx", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] BatchJobEx(Roblox.Grid.Rcc.Server.Job job, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            try
+            {
+                if (Properties.Settings.Default.Verbose) Console.WriteLine("BatchJob(" + job.ToString() + ")");
+                string jobId = "Batch" + job.id;
+                Roblox.Grid.Rcc.Server.LuaValue[] result;
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.NewJob(jobId, job.expirationInSeconds, "BatchJob"))
+                {
+                    try
+                    {
+                        result = convert(rccService.BatchJob(convert(job), convert(script)));
+                    }
+                    finally
+                    {
+                        jobManager.CloseJob(jobId);
+                    }
+                }
+                Log.Event(String.Format("BatchJobEx('{0}', {1}, {2}) - Success", job.id, job.category, job.expirationInSeconds));
+                return result;
+
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("BatchJobEx('{0}', {1}, {2}) - Exception - {3}", job.id, job.category, job.expirationInSeconds, e.Message));
+                throw;
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/RenewLease", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override double RenewLease(string jobID, double expirationInSeconds)
+        {
+            System.Threading.ThreadPool.QueueUserWorkItem((dummy) =>
+            {
+                try
+                {
+                    RenewLeaseSync(jobID, expirationInSeconds);                    
+                }
+                catch (Exception ex)
+                {
+                    ExceptionHandler.LogException(ex);                        
+                }
+            });
+            return 0;
+        }
+        private void RenewLeaseSync(string jobID, double expirationInSeconds)
+        {
+            try
+            {
+                if (Properties.Settings.Default.Verbose) Console.WriteLine("RenewLease(" + jobID + "," + expirationInSeconds + ")");
+                jobManager.RenewLease(jobID, expirationInSeconds);
+                double result;
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.GetJob(jobID, "RenewLease", ""))
+                {
+                    result = rccService.RenewLease(jobID, expirationInSeconds);
+                }
+                Log.Event(String.Format("RenewLease('{0}', {1}) - Success", jobID, expirationInSeconds));                
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("RenewLease('{0}', {1}) - Exception - {2}", jobID, expirationInSeconds, e.Message));
+                throw;
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/Execute", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] Execute(string jobID, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            throw new Exception("Execute is deprecated, use ExecuteEx");
+        }
+       
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/ExecuteEx", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] ExecuteEx(string jobID, Roblox.Grid.Rcc.Server.ScriptExecution script)
+        {
+            try
+            {
+                if (Properties.Settings.Default.Verbose) Console.WriteLine("Execute(" + jobID + ")");
+
+                Roblox.Grid.Rcc.Server.LuaValue[] finalResult;
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.GetJob(jobID, "ExecuteEx", script.script.Substring(0, Math.Min(30, script.script.Length))))
+                {
+                    Roblox.Grid.Rcc.LuaValue[] result = rccService.Execute(jobID, convert(script));
+                    finalResult = convert(result);
+                }
+                Log.Event(String.Format("ExecuteEx('{0}', {1}) - Success", jobID, script.name));
+                return finalResult;
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("ExecuteEx('{0}', {1}) - Exception - {2}", jobID, script.name, e.Message));
+                throw;
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/CloseJob", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override void CloseJob(string jobID)
+        {
+            System.Threading.ThreadPool.QueueUserWorkItem((dummy) =>
+            {
+                try
+                {
+                    CloseJobSync(jobID);
+                }
+                catch (Exception ex)
+                {
+                    ExceptionHandler.LogException(ex);
+                }
+            });            
+        }
+        
+        public void CloseJobSync(string jobID)
+        {
+            try
+            {
+                if (Properties.Settings.Default.Verbose) Console.WriteLine("CloseJob(" + jobID + ")");
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.GetJob(jobID, "CloseJob", ""))
+                {
+                    rccService.CloseJob(jobID);
+                }
+            }
+            catch (Exception e)
+            {
+                Log.Event(String.Format("CloseJob('{0}') - Exception - {1}", jobID, e.Message));
+                throw;
+            }
+            finally
+            {
+                jobManager.CloseJob(jobID);
+            }
+            Log.Event(String.Format("CloseJob('{0}') - Success", jobID));
+        }
+
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/GetExpiration", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override double GetExpiration(string jobID)
+        {
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("GetExpiration(" + jobID + ")");
+            using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.GetJob(jobID, "GetExpiration", ""))
+            {
+                return rccService.GetExpiration(jobID);
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/Diag", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] Diag(int type, string jobID)
+        {
+            throw new Exception("Diag is deprecated, use DiagEx");
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/DiagEx", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.LuaValue[] DiagEx(int type, string jobID)
+        {
+
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("Diag(" +    type + "," + jobID + ")");
+            if (jobID != "")
+            {
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.GetJob(jobID, "DiagEx", ""))
+                {
+                    return convert(rccService.Diag(type, jobID));
+                }
+            }
+            else
+            {
+                using (Roblox.Grid.Rcc.RCCServiceSoap rccService = jobManager.AnyJob(60))
+                {
+                    return convert(rccService.Diag(type, jobID));
+                }
+            }
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/GetStatus", ReplyAction = "http://watrbx.wtf/GetStatusResponse")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.Status GetStatus()
+        {
+            //if (Properties.Settings.Default.Verbose) Console.WriteLine("GetStatus()");
+            Roblox.Grid.Rcc.Server.Status result = new Roblox.Grid.Rcc.Server.Status();
+            result.environmentCount = 0;
+
+            //Pick version from 1
+            result.version = GetVersion();
+
+            jobManager.DispatchRequest(delegate(Roblox.Grid.Rcc.RCCServiceSoap rccService)
+                {
+                    //Call GetStatus on all dependent processes
+                    Roblox.Grid.Rcc.Status status = rccService.GetStatus();
+
+                    //Ensure all have same version
+                    Debug.Assert(status.version != result.version);
+
+                    //Sum environmentCount from each process
+                    result.environmentCount += status.environmentCount;
+                });
+            return result;
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/GetAllJobs", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.Job[] GetAllJobs()
+        {
+            throw new Exception("GetAllJobs is deprecated, use GetAllJobsEx");
+        }
+
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/GetAllJobsEx", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override Roblox.Grid.Rcc.Server.Job[] GetAllJobsEx()
+        {
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("GetAllJobs()");
+            System.Collections.Generic.List<Roblox.Grid.Rcc.Server.Job> result = new List<Roblox.Grid.Rcc.Server.Job>();
+
+            jobManager.DispatchRequest(delegate(Roblox.Grid.Rcc.RCCServiceSoap rccService)
+            {
+                Roblox.Grid.Rcc.Job[] jobs = rccService.GetAllJobs();
+                foreach (Roblox.Grid.Rcc.Job job in jobs)
+                    result.Add(convert(job));
+            });
+            return result.ToArray();
+    }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/CloseExpiredJobs", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override int CloseExpiredJobs()
+        {
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("CloseExpiredJobs()");
+            int result = 0;
+            //Forward to all processes
+            jobManager.DispatchRequest(delegate(Roblox.Grid.Rcc.RCCServiceSoap rccService)
+                {
+                    result += rccService.CloseExpiredJobs();
+                });
+
+            return result;
+        }
+
+        [System.ServiceModel.OperationContractAttribute(Action = "http://watrbx.wtf/CloseAllJobs", ReplyAction = "*")]
+        [System.ServiceModel.XmlSerializerFormatAttribute()]
+        public override int CloseAllJobs()
+        {
+            if (Properties.Settings.Default.Verbose) Console.WriteLine("CloseAllJobs()");
+            int result = 0;
+            //Forward to all processes
+            jobManager.DispatchRequest(delegate(Roblox.Grid.Rcc.RCCServiceSoap rccService)
+                {
+                    result += rccService.CloseAllJobs();
+                });
+
+            return result;
+        }        
+        #endregion
+    }
+}

Roblox.RccServiceArbiter/RccServiceMonitor.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.ServiceModel;
+
+namespace Roblox.RccServiceArbiter
+{
+    
+    [ServiceBehavior(InstanceContextMode = InstanceContextMode.Single, ConcurrencyMode = ConcurrencyMode.Multiple)]
+    class RccServiceMonitor :
+        IArbiter
+    {
+        RccService rccService;
+
+        public RccServiceMonitor(RccService rccService)
+        {
+            this.rccService = rccService;
+        }
+
+        public string GetStats()
+        {
+            return rccService.GetStats(false);
+        }
+
+        public string GetStatsEx(bool clearExceptions)
+        {
+            return rccService.GetStats(clearExceptions);
+        }
+
+        public void SetMultiProcess(bool value)
+        {
+            rccService.SetMultiProcess(value);
+        }
+
+        public void SetRecycleProcess(bool value)
+        {
+            rccService.SetRecycleProcess(value);
+        }
+
+        public void SetRecycleQueueSize(int value)
+        {
+            rccService.SetRecycleProcessCount(value);
+        }
+
+        public void SetThreadConfigScript(string script, bool broadcast)
+        {
+            rccService.SetThreadConfigScript(script, broadcast);
+        }
+    }
+}

Roblox.RccServiceArbiter/Roblox.RccServiceArbiter.csproj

@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>9.0.30729</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{2B5528CA-849E-4826-96F7-10725C39715A}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Roblox.RccServiceArbiter</RootNamespace>
+    <AssemblyName>Roblox.RccServiceArbiter</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <SccProjectName>SAK</SccProjectName>
+    <SccLocalPath>SAK</SccLocalPath>
+    <SccAuxPath>SAK</SccAuxPath>
+    <SccProvider>SAK</SccProvider>
+    <FileUpgradeFlags>
+    </FileUpgradeFlags>
+    <UpgradeBackupLocation>
+    </UpgradeBackupLocation>
+    <OldToolsVersion>3.5</OldToolsVersion>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseStudio|AnyCPU' ">
+    <OutputPath>bin\ReleaseStudio\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <CodeAnalysisUseTypeNameInSuppression>true</CodeAnalysisUseTypeNameInSuppression>
+    <CodeAnalysisModuleSuppressionsFile>GlobalSuppressions.cs</CodeAnalysisModuleSuppressionsFile>
+    <ErrorReport>prompt</ErrorReport>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Configuration.Install" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.ServiceModel">
+      <RequiredTargetFramework>3.0</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.ServiceProcess" />
+    <Reference Include="System.Web" />
+    <Reference Include="System.Web.Extensions">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Web.Services" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ArbiterInstaller.cs">
+      <SubType>Component</SubType>
+    </Compile>
+    <Compile Include="IArbiter.cs" />
+    <Compile Include="JobManager.cs" />
+    <Compile Include="Log.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Properties\Settings.Designer.cs">
+      <AutoGen>True</AutoGen>
+      <DesignTimeSharedInput>True</DesignTimeSharedInput>
+      <DependentUpon>Settings.settings</DependentUpon>
+    </Compile>
+    <Compile Include="RccService.cs">
+      <SubType>Component</SubType>
+    </Compile>
+    <Compile Include="RccServiceMonitor.cs" />
+    <Compile Include="TcpPort.cs" />
+    <Compile Include="Util.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <WCFMetadata Include="Service References\" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="app.config" />
+    <None Include="Properties\Settings.settings">
+      <Generator>SettingsSingleFileGenerator</Generator>
+      <LastGenOutput>Settings.Designer.cs</LastGenOutput>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\Client.LegacyAssemblies\Grid\Arbiter\Roblox.Grid.Arbiter.Common\Roblox.Grid.Arbiter.Common.csproj">
+      <Project>{A8540F3F-9CEB-437E-AE59-EBCFE4014051}</Project>
+      <Name>Roblox.Grid.Arbiter.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Client.LegacyAssemblies\Grid\Roblox.Grid.Common\Roblox.Grid.Common.csproj">
+      <Project>{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}</Project>
+      <Name>Roblox.Grid.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Client.LegacyAssemblies\Roblox.Common\Roblox.Common.csproj">
+      <Project>{31C4415B-A946-47BD-83FE-AE13FBFB1324}</Project>
+      <Name>Roblox.Common</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

Roblox.RccServiceArbiter/Roblox.RccServiceArbiter.sln

@@ -0,0 +1,71 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.36227.6
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.RccServiceArbiter", "Roblox.RccServiceArbiter.csproj", "{2B5528CA-849E-4826-96F7-10725C39715A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.Common", "..\..\Client.LegacyAssemblies\Roblox.Common\Roblox.Common.csproj", "{31C4415B-A946-47BD-83FE-AE13FBFB1324}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.Grid.Client", "..\..\Client.LegacyAssemblies\Grid\Roblox.Grid.Client\Roblox.Grid.Client.csproj", "{C4995CBC-3218-4651-8757-5F315822F09F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.Configuration", "..\..\Client.LegacyAssemblies\Roblox.Configuration\Roblox.Configuration.csproj", "{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.Grid.Common", "..\..\Client.LegacyAssemblies\Grid\Roblox.Grid.Common\Roblox.Grid.Common.csproj", "{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Roblox.Grid.Arbiter.Common", "..\..\Client.LegacyAssemblies\Grid\Arbiter\Roblox.Grid.Arbiter.Common\Roblox.Grid.Arbiter.Common.csproj", "{A8540F3F-9CEB-437E-AE59-EBCFE4014051}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B5528CA-849E-4826-96F7-10725C39715A}.Release|x86.ActiveCfg = Release|Any CPU
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Debug|x86.ActiveCfg = Debug|x86
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Debug|x86.Build.0 = Debug|x86
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Release|Any CPU.Build.0 = Release|Any CPU
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Release|x86.ActiveCfg = Release|x86
+		{31C4415B-A946-47BD-83FE-AE13FBFB1324}.Release|x86.Build.0 = Release|x86
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C4995CBC-3218-4651-8757-5F315822F09F}.Release|x86.ActiveCfg = Release|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D996AD59-D9C6-4DBB-9E96-D7C90896E8D0}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC7F620E-40D7-41F9-A569-2401AB1B9EAF}.Release|x86.ActiveCfg = Release|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A8540F3F-9CEB-437E-AE59-EBCFE4014051}.Release|x86.ActiveCfg = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {3DFDE8A8-C1FF-4CA1-8AE1-8EBF1706DBA4}
+	EndGlobalSection
+EndGlobal

Roblox.RccServiceArbiter/TcpPort.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Roblox.RccServiceArbiter
+{
+    /// <summary>
+    /// Taken from http://www.mattbrindley.com/developing/windows/net/detecting-the-next-available-free-tcp-port/
+    /// </summary>
+    public static class TcpPort
+    {
+        private const string PortReleaseGuid = "8875BD8E-4D5B-11DE-B2F4-691756D89593";
+
+        /// <summary>
+        /// Check if startPort is available, incrementing and
+        /// checking again if it's in use until a free port is found
+        /// </summary>
+        /// <param name="startPort">The first port to check</param>
+        /// <returns>The first available port</returns>
+        public static int FindNextAvailablePort(int startPort)
+        {
+            int port = startPort;
+            bool isAvailable = true;
+
+            var mutex = new System.Threading.Mutex(false,
+                string.Concat("Global/", PortReleaseGuid));
+            mutex.WaitOne();
+            try
+            {
+                System.Net.NetworkInformation.IPGlobalProperties ipGlobalProperties =
+                    System.Net.NetworkInformation.IPGlobalProperties.GetIPGlobalProperties();
+                System.Net.IPEndPoint[] endPoints =
+                    ipGlobalProperties.GetActiveTcpListeners();
+
+                do
+                {
+                    if (!isAvailable)
+                    {
+                        port++;
+                        isAvailable = true;
+                    }
+
+                    foreach (System.Net.IPEndPoint endPoint in endPoints)
+                    {
+                        if (endPoint.Port != port) continue;
+                        isAvailable = false;
+                        break;
+                    }
+
+                } while (!isAvailable && port < System.Net.IPEndPoint.MaxPort);
+
+                if (!isAvailable)
+                    throw new Exception("NoAvailablePortsInRangeException");
+
+                return port;
+            }
+            finally
+            {
+                mutex.ReleaseMutex();
+            }
+        }
+    }
+}

Roblox.RccServiceArbiter/Util.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Net;
+using System.IO;
+using System.Xml;
+
+namespace Roblox.RccServiceArbiter
+{
+    class ClientSettings
+    {
+        static private string BuildSettingsUrl(string baseUrl, string group)
+        {
+            UriBuilder uriBuild = new UriBuilder(baseUrl);
+            string host = uriBuild.Host;
+            host = host.Replace("www.", "");
+
+            return String.Format("https://clientsettings.api.{0}/Setting/QuietGet/{1}/?apiKey=D6925E56-BFB9-4908-AAA2-A5B1EC4B2D79", host, group);;
+        }
+
+        static public string Fetch(string group)
+        {
+            string baseUrl = Utils.GetBaseURL();
+            if (baseUrl.Length == 0)
+            {
+                // You didn't set BaseURL before loading settings!
+                Console.WriteLine("Failed to get base url");
+                return null;
+            }
+
+            string settingsData = "";
+            try
+            {
+                string url = BuildSettingsUrl(baseUrl, group);
+                
+                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
+                
+                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+                if (response.ContentLength > 0)
+                {
+                    StreamReader readStream = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
+                    settingsData = readStream.ReadToEnd();
+                }
+                response.Close();
+            }
+            catch (Exception exp)
+            {
+                Console.WriteLine(exp.Message);
+                settingsData = null;
+            }
+
+            return settingsData;
+        }
+
+        static public string FetchThreadPoolConfig()
+        {
+            string jsonData = Fetch("Arbiter");
+
+            if (jsonData != null)
+            {
+                Dictionary<string, object> json = new System.Web.Script.Serialization.JavaScriptSerializer().Deserialize<Dictionary<string, object>>(jsonData);
+
+                object value;
+                if (json.TryGetValue("ThreadPoolConfig", out value))
+                {
+                    if (value != null)
+                        return value.ToString();
+                }
+            }
+
+            return null;
+        }
+    };
+
+    class Utils
+    {
+        static private string baseUrl = String.Empty;
+
+        static public string GetBaseURL()
+        {
+            if (baseUrl.Length == 0)
+            {
+                System.Reflection.Assembly assembly = System.Reflection.Assembly.GetExecutingAssembly();
+                string fullProcessPath = assembly.Location;
+                string settingsFilePath = System.IO.Path.GetDirectoryName(fullProcessPath) + System.IO.Path.DirectorySeparatorChar + Properties.Settings.Default.AppSettingPath;
+
+                XmlDocument xml = new XmlDocument();
+                xml.Load(settingsFilePath);
+                XmlNodeList nodes = xml.GetElementsByTagName("BaseUrl");
+                baseUrl = nodes[0].InnerText;
+            }
+            return baseUrl;
+        }
+    }
+}

Roblox.RccServiceArbiter/app.config

@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<configuration>
+	<configSections>
+		<sectionGroup name="userSettings" type="System.Configuration.UserSettingsGroup, System, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+			<section name="Roblox.RccServiceArbiter.Properties.Settings" type="System.Configuration.ClientSettingsSection, System, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" allowExeDefinition="MachineToLocalUser" requirePermission="false"/>
+		</sectionGroup>
+	</configSections>
+	<system.serviceModel>
+		<bindings>
+			<basicHttpBinding>
+				<binding name="GridBindingConfiguration" maxBufferPoolSize="524288" maxBufferSize="5242880" maxReceivedMessageSize="5242880">
+					<readerQuotas maxStringContentLength="5242880" maxArrayLength="5242880" maxBytesPerRead="4096" maxDepth="32" maxNameTableCharCount="16384"/>
+				</binding>
+			</basicHttpBinding>
+		</bindings>
+		<services>
+			<service name="Roblox.RccServiceArbiter.RccService" behaviorConfiguration="gridBehaviorConfiguration">
+				<host>
+					<baseAddresses>
+						<add baseAddress="http://localhost:64989"/>
+					</baseAddresses>
+				</host>
+				<endpoint name="RccService" address="" binding="basicHttpBinding" bindingConfiguration="GridBindingConfiguration" contract="RCCServiceSoap"/>
+			</service>
+			<service name="Roblox.RccServiceArbiter.RccServiceMonitor" behaviorConfiguration="gridBehaviorConfiguration">
+				<host>
+					<baseAddresses>
+						<add baseAddress="http://localhost:64990"/>
+					</baseAddresses>
+				</host>
+				<endpoint name="RccServiceMonitor" address="" binding="basicHttpBinding" bindingConfiguration="GridBindingConfiguration" contract="IArbiter"/>
+			</service>
+		</services>
+		<behaviors>
+			<serviceBehaviors>
+				<behavior name="gridBehaviorConfiguration">
+					<serviceThrottling maxConcurrentCalls="2147483647" maxConcurrentInstances="2147483647" maxConcurrentSessions="2147483647"/>
+					<serviceMetadata/>
+					<serviceDebug includeExceptionDetailInFaults="true"/>
+				</behavior>
+			</serviceBehaviors>
+		</behaviors>
+	</system.serviceModel>
+	<userSettings>
+		<Roblox.RccServiceArbiter.Properties.Settings>
+			<setting name="RecycleProcesses" serializeAs="String">
+				<value>False</value>
+			</setting>
+			<setting name="SingleProcess" serializeAs="String">
+				<value>False</value>
+			</setting>
+			<setting name="Verbose" serializeAs="String">
+				<value>True</value>
+			</setting>
+			<setting name="RccServicePath" serializeAs="String">
+				<value>RCCService.exe</value>
+			</setting>
+			<setting name="RccServiceLaunch" serializeAs="String">
+				<value>/content:content\\</value>
+			</setting>
+			<setting name="LogTransactions" serializeAs="String">
+				<value>False</value>
+			</setting>
+			<setting name="Timeout" serializeAs="String">
+				<value>00:05:00</value>
+			</setting>
+			<setting name="MaxConnections" serializeAs="String">
+				<value>128</value>
+			</setting>
+			<setting name="StartScript" serializeAs="String">
+				<value>
+          settings()['Task Scheduler'].ThreadPoolConfig = Enum.ThreadPoolConfig.Threads1;
+          -- deprecated settings()['Task Scheduler']:SetThreadShare(1000,4);
+        </value>
+			</setting>
+			<setting name="AppSettingPath" serializeAs="String">
+				<value>AppSettings.xml</value>
+			</setting>
+		</Roblox.RccServiceArbiter.Properties.Settings>
+	</userSettings>
+	<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.0"/></startup>
+	<system.net>
+		<connectionManagement>
+			<add address="*" maxconnection="2147483647"/>
+		</connectionManagement>
+	</system.net>
+</configuration>