#define USE_REMOTE_CLIENTS using System; using System.Text; using System.Collections.Generic; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using Roblox.Grid.Rcc; using Roblox.Grid; using System.IO; using System.Threading; using System.Diagnostics; namespace RCCService.Test { /// /// Summary description for StressTest /// [TestClass] public class StressTests { public StressTests() { // // TODO: Add constructor logic here // } private TestContext testContextInstance; /// ///Gets or sets the test context which provides ///information about and functionality for the current test run. /// public TestContext TestContext { get { return testContextInstance; } set { testContextInstance = value; } } private class GameInfo { public GameInfo(int gamenum, string id, Uri gameUri, bool active) { this.gamenum = gamenum; this.id = id; this.active = active; this.gameUri = gameUri; } public int gamenum; public string id; public bool active; public Uri gameUri; }; class ClientManager : IDisposable { int servicenum; private Uri uri; private RCCServiceSoap service; private List games; public static int webRequestTimeout = 60* 1000; public static bool ignoreWebExceptions = false; // set this to true for tests where you expect occasionnal client downs. public static List CreateNRemoteClients(int firstclientnum, int numclients) { List result = new List(); List clientUris = new List(); #if USE_REMOTE_CLIENTS clientUris = RCCLauncher.FindNMoreRemoteRCCServices(numclients); #else for (int i = 0; i < numclients; ++i) { clientUris.Add(RCCLauncher.StartRCCService()); } #endif foreach (Uri uri in clientUris) { result.Add(new ClientManager(firstclientnum++, uri)); } return result; } public ClientManager(int servicenum, Uri uri) { this.servicenum = servicenum; this.uri = uri; this.games = new List(); } public void AquireService() { service = Roblox.Grid.Common.GridServiceUtils.GetService(uri.Host, uri.Port); service.Timeout = webRequestTimeout; service.CloseAllJobs(); // cleanup potential crumbs from failed previous run } public void ReAquireServiceAndGames() { this.uri = RCCLauncher.FindRemoteRCCService(); AquireService(); // try to re-join all the games that we used to be in foreach (GameInfo gi in games) { Roblox.Grid.Rcc.Job job = new Roblox.Grid.Rcc.Job(); job.expirationInSeconds = Math.Max(2 * 60 * 60, webRequestTimeout / 1000); job.category = 0; job.id = gi.id; service.OpenJob(job, LuaScripts.MakeJoinGameScript(gi.gameUri.Host/*Lists.routerAddress*/, gi.gameUri.Port)); gi.active = false; } } public void EnsureUp() { if (!RCCLauncher.IsServiceAvailable(uri.Host, uri.Port, 0)) { Stop(); ReAquireServiceAndGames(); } } private void CloseGameWorker(GameInfo game) { try { service.CloseJob(game.id); } catch (System.Net.WebException) { if (!ignoreWebExceptions) { throw; } } } // close job connected to game if exists (returns true if exists) public bool CloseGame(int gamenum) { bool b = false; for (int i = games.Count-1; i != -1; --i) { if (games[i].gamenum == gamenum) { CloseGameWorker(games[i]); games.RemoveAt(i); b = true; } } return b; } private void JoinGameWorker(GameInfo newgame) { Roblox.Grid.Rcc.Job job = new Roblox.Grid.Rcc.Job(); job.expirationInSeconds = Math.Max(2 * 60 * 60, webRequestTimeout / 1000); job.category = 0; job.id = newgame.id; try { service.OpenJob(job, LuaScripts.MakeJoinGameScript(newgame.gameUri.Host/*Lists.routerAddress*/, newgame.gameUri.Port)); } catch (System.Net.WebException) { if (!ignoreWebExceptions) { throw; } } } public GameInfo JoinGame(GameInfo servergame, int clientnum) { GameInfo newgame = new GameInfo( servergame.gamenum, servergame.id + "_Client#" + clientnum.ToString(), servergame.gameUri, false); // we use active for wether or not the busy script is running. games.Add(newgame); JoinGameWorker(newgame); return newgame; } public void ActivateInactiveGames(ScriptExecution scriptExecution) { foreach (GameInfo gi in games) { if (gi.active) { continue; } try { service.Execute(gi.id, scriptExecution); gi.active = true; } catch (System.Net.WebException) { if (!ignoreWebExceptions) { throw; } } catch (Exception) { // not critical } } } public void Stop() { if (uri != null) { RCCLauncher.StopRCCService(uri); uri = null; } } #region IDisposable Members public void Dispose() { Stop(); } #endregion public void Execute(string jobid, ScriptExecution scriptExecution) { try { service.Execute(jobid, scriptExecution); } catch (System.Net.WebException) { if (!ignoreWebExceptions) { throw; } } } public int GameCount() { return games.Count; } public void ToggleGame(int gamei) { CloseGameWorker(games[gamei]); games[gamei].active = false; JoinGameWorker(games[gamei]); } }; private Uri serverUri; private List rccclients = new List(); [TestInitialize()] public void MyTestInitialize() { // complex multi-rcc server test required clean slate. RCCLauncher.StopAllRCCServices(); //serverUri = new Uri(""); rccclients.Clear(); } [TestCleanup()] public void MyTestCleanup() { RCCLauncher.StopRCCService(serverUri); foreach (ClientManager c in rccclients) { c.Stop(); } rccclients.Clear(); } private delegate void CloseJobDelegateType(int gamenum); public delegate ScriptExecution ServerScriptDelegateType(int placerun); public static void DefaultWaitDelegate(object delegateParams) { foreach (ClientManager cm in ((StressTests)delegateParams).rccclients) { cm.EnsureUp(); cm.ActivateInactiveGames(LuaScripts.MakeBusyScript()); } } public class StressClientServerParams { public int numGamesOnServer = 1; public int numClients = 8; public int numRCCClients = 4; public int numPlacesToLoad = 1; public int waitMSEachPlace = 60 * 1000; public int webRequestTimeout = defaultWebRequestTimeout; public bool allowFailingClients = false; public ServerScriptDelegateType scriptChooser = null; public TestUtils.waitDelegate waitDelegate = DefaultWaitDelegate; public int samplePeriodms = 60 * 1000; // also corresponds to frequency of call to waitDelegate. public StressClientServerParams() { } public StressClientServerParams(int numGamesOnServer, int numClients, int numRCCClients, int numPlacesToLoad, int waitMSEachPlace, int webRequestTimeout, bool allowFailingClients, ServerScriptDelegateType scriptChooser) { this.numGamesOnServer = numGamesOnServer; this.numClients = numClients; this.numRCCClients = numRCCClients; this.numPlacesToLoad = numPlacesToLoad; this.waitMSEachPlace = waitMSEachPlace; this.webRequestTimeout = webRequestTimeout; this.allowFailingClients = allowFailingClients; this.scriptChooser = scriptChooser; } public StressClientServerParams(int numGamesOnServer, int numClients, int numRCCClients, int numPlacesToLoad, int waitMSEachPlace, int webRequestTimeout, bool allowFailingClients, int[] placeAssetIds) { this.numGamesOnServer = numGamesOnServer; this.numClients = numClients; this.numRCCClients = numRCCClients; this.numPlacesToLoad = numPlacesToLoad; this.waitMSEachPlace = waitMSEachPlace; this.webRequestTimeout = webRequestTimeout; this.allowFailingClients = allowFailingClients; this.placeAssetIds = placeAssetIds; } public int[] placeAssetIds { set { scriptChooser = delegate(int placerun) { return LuaScripts.MakeLoadPlaceScript(value[placerun % value.Length]); }; } } } public void StressClientServer(StressClientServerParams p) { TestContext.WriteLine("Simultaneous games on server: {0}", p.numGamesOnServer); TestContext.WriteLine("Number of clients connected: {0}", p.numClients); TestContext.WriteLine("Number of RCCClients spawned: {0}", p.numRCCClients); TestContext.WriteLine("Number of places loaded: {0}", p.numPlacesToLoad); TestContext.WriteLine("Wait in running game in milliseconds: {0}", p.waitMSEachPlace); //Debug.Assert(numRCCClients >= numClients / numGamesOnServer, "number of spawend RCCClients must equal or exceed the number of clients per games. (so that the same RCCClient won't connect twice on the same game)"); serverUri = RCCLauncher.StartRCCService(); ClientManager.webRequestTimeout = p.webRequestTimeout; ClientManager.ignoreWebExceptions = p.allowFailingClients; rccclients = ClientManager.CreateNRemoteClients(0, p.numRCCClients); int gameport = 53640; List serverjobids = new List(p.numGamesOnServer); for (int i = 0; i < p.numGamesOnServer; ++i) { UriBuilder urib = new UriBuilder(serverUri); urib.Port = gameport++; serverjobids.Add(new GameInfo(i, "", urib.Uri, false)); } int currentgame = 0; // cycle commands through each game server. RCCServiceSoap serverService = null; CloseJobDelegateType closeJobDelegate = delegate(int gamenum) { foreach (ClientManager c in rccclients) { c.CloseGame(gamenum); } serverService.CloseJob(serverjobids[gamenum].id); serverjobids[gamenum].active = false; }; try // try/finalluy block to make sure we dispose the serverService and the clientService { serverService = Roblox.Grid.Common.GridServiceUtils.GetService(serverUri.Host, serverUri.Port); serverService.Timeout = p.webRequestTimeout; serverService.CloseAllJobs(); // cleanup potential crumbs from failed previous run foreach (ClientManager c in rccclients) { c.AquireService(); } Roblox.Grid.Rcc.Job job = new Roblox.Grid.Rcc.Job(); job.expirationInSeconds = Math.Max(2 * 60 * 60, p.webRequestTimeout / 1000); job.category = 0; TestContext.WriteLine("PrivateWorkingSetBytes: run#, workingset bytes, bytes, cores"); for (int placerun = 0; placerun < p.numPlacesToLoad; placerun++) { LuaValue[] result = null; if (serverjobids[currentgame].active == true) { closeJobDelegate(currentgame); } // save job ID so we can use it now, and close it later. serverjobids[currentgame].id = "TestSimpleClientServer_game" + currentgame.ToString() + "_placerun" + placerun.ToString(); job.id = serverjobids[currentgame].id; result = serverService.OpenJob(job, Lua.EmptyScript); serverjobids[currentgame].active = true; result = serverService.Execute(serverjobids[currentgame].id, p.scriptChooser(placerun)); result = serverService.Execute(serverjobids[currentgame].id, Lua.NewScript("StartGameServer", LuaScripts.StartGameServer(), serverjobids[currentgame].gameUri.Port)); // 3568125 // empty place: 381342 //connect clients, partition client pool so that we don't reallocate _All_ the clients to the most recently opened game. // on a 4 game server, with 16 clients: // clients 0, 1, 2, 4 would go on game 0 // clients 5, 6, 7, 8 would go on game 1, etc... int expectedClientCount = 0; Dictionary clientGames = new Dictionary(); for (int clienti = (p.numClients * currentgame) / p.numGamesOnServer; clienti < (p.numClients * (currentgame + 1)) / p.numGamesOnServer; ++clienti) { int wrappedClienti = clienti % rccclients.Count(); clientGames[wrappedClienti] = rccclients[wrappedClienti].JoinGame(serverjobids[currentgame], clienti); } foreach (ClientManager cm in rccclients) { cm.EnsureUp(); } TestUtils.WaitForNClients(serverService, serverjobids[currentgame].id, clientGames.Count, 15 * 1000); // only do the full wait once the pipeline is full. int waitms = (placerun + 1 >= p.numGamesOnServer) ? p.waitMSEachPlace : 0; TestUtils.PrintStatisticsAndWait(serverService, serverjobids[currentgame].id, TestContext, placerun, waitms / p.samplePeriodms, p.samplePeriodms, p.waitDelegate, this); foreach (ClientManager cm in rccclients) { cm.EnsureUp(); } // check to make sure clients are still connected. TestUtils.WaitForNClients(serverService, serverjobids[currentgame].id, expectedClientCount, 0); // cycle through each game on the server. currentgame = (currentgame + 1) % p.numGamesOnServer; } //for // cleanup for (int i = 0; i < p.numGamesOnServer; ++i) { closeJobDelegate(i); } // for } finally { serverService.Dispose(); for (int clienti = 0; clienti < rccclients.Count; ++clienti) { rccclients[clienti].Dispose(); } } } // func #region Additional test attributes // // You can use the following additional attributes as you write your tests: // // Use ClassInitialize to run code before running the first test in the class // [ClassInitialize()] // public static void MyClassInitialize(TestContext testContext) { } // // Use ClassCleanup to run code after all tests in a class have run // [ClassCleanup()] // public static void MyClassCleanup() { } // // Use TestInitialize to run code before running each test // [TestInitialize()] // public void MyTestInitialize() { } // // Use TestCleanup to run code after each test has run // [TestCleanup()] // public void MyTestCleanup() { } // #endregion private const int defaultWebRequestTimeout = 60 * 1000; [TestMethod] public void TestStressLoadManyPopularPlaces() { StressClientServer( new StressClientServerParams(1, 0, 0, 200, 0, defaultWebRequestTimeout, false, Lists.popularPlaces)); } [TestMethod] public void TestStress4Games16Clients() { StressClientServer( new StressClientServerParams(4, 16, 4, 4, 0, defaultWebRequestTimeout, false, Lists.popularPlaces)); } [TestMethod] [Timeout(2*60*60*1000)] // 2 hours public void TestStress4Games16ClientsOverManyPopularPlaces() { StressClientServer( new StressClientServerParams(4, 16, 4, 200, 0, defaultWebRequestTimeout, false, Lists.popularPlaces)); } [TestMethod] [Timeout(2 * 60 * 60 * 1000)] // 2 hours public void TestStress8Games8ClientsOverTop25() { StressClientServer( new StressClientServerParams(8, 8, 1, 100, 0, defaultWebRequestTimeout * 4, false, Lists.top25)); } [TestMethod] [Timeout(2 * 60 * 60 * 1000)] // 2 hours public void TestStress8Games16ClientsOverTop25() { StressClientServer( new StressClientServerParams(8, 16, 4, 100, 0, defaultWebRequestTimeout * 4, false, Lists.top25)); } [TestMethod] [Timeout(2 * 60 * 60 * 1000)] // 2 hours public void TestStress8Games32ClientsOverTop25() { StressClientServer( new StressClientServerParams(8, 32, 4, 100, 0, defaultWebRequestTimeout * 4, false, Lists.top25)); } [TestMethod] [Timeout(2 * 60 * 60 * 1000)] // 2 hours public void TestStress8Games64ClientsOverTop25() { StressClientServer( new StressClientServerParams(8, 64, 8, 100, 0, defaultWebRequestTimeout * 4, false, Lists.top25)); } [TestMethod] public void TestStressPhysicsBenchmarkShort() { StressClientServer( new StressClientServerParams(4, 8, 2, 4, 15 * 1000, defaultWebRequestTimeout * 2, false, new int[] { 4420793 })); } [TestMethod] public void TestStress1Games8ClientsMTBF() { StressClientServer(new StressClientServerParams(1, 8, 8, 1, 10 * 60 * 60 * 1000 /*10 hours*/, defaultWebRequestTimeout * 2, false, new int[] { 7074883 })); } [TestMethod] [Timeout(20 * 60 * 60 * 1000)] // 20 hours public void TestStress8Games128ClientsMTBF() { StressClientServer( new StressClientServerParams(8, 128, 16, 1, 10 * 60 * 60 * 1000 /*10 hours*/, defaultWebRequestTimeout * 2, true, new int[] { 7074883 })); } [TestMethod] [Timeout(20 * 60 * 60 * 1000)] // 20 hours public void TestStress8Games128ClientsConnectDisconnect() { Random rand = new Random(); StressClientServerParams p = new StressClientServerParams(); p.placeAssetIds = new int[] { 7074883 }; p.waitMSEachPlace = 60 * 60 * 1000; /*1 hours*/ p.numClients = 128; p.numRCCClients = 16; int minutesConnectedPerClient = 3; p.samplePeriodms = minutesConnectedPerClient * 60 * 1000 / p.numClients; p.waitDelegate = delegate(object waitParams) { // do the usual first StressTests pthis = (StressTests)waitParams; DefaultWaitDelegate(pthis); // randomly disconnect and reconnect a client every p.samplePreiodms. // equivalent to having every client be connected on average "minutesConnectedPerClient" int rccclientnum = rand.Next(pthis.rccclients.Count); int gamenum = rand.Next(pthis.rccclients[rccclientnum].GameCount()); pthis.rccclients[rccclientnum].ToggleGame(gamenum); //pthis.rccclients[rccclientnum]. }; StressClientServer(p); } } }