From dd4c8dc3aaa3b556bce4ff519eb98dd8cb53333c Mon Sep 17 00:00:00 2001
From: Peter 'Pita' Martischka <petermartischka@googlemail.com>
Date: Thu, 18 Aug 2011 20:58:56 +0100
Subject: [PATCH] added a convert script from old etherpad to etherpad lite,
 instructions will follow

---
 .gitignore                        |   1 +
 bin/convert.js                    | 449 ++++++++++++++++++++++++++++++
 bin/convertSettings.json.template |  21 ++
 3 files changed, 471 insertions(+)
 create mode 100644 bin/convert.js
 create mode 100644 bin/convertSettings.json.template

diff --git a/.gitignore b/.gitignore
index 29835e6f..ab91434b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ bin/abiword.exe
 bin/node.exe
 etherpad-lite-win.zip
 var/dirty.db
+bin/convertSettings.json
\ No newline at end of file
diff --git a/bin/convert.js b/bin/convert.js
new file mode 100644
index 00000000..94e9877a
--- /dev/null
+++ b/bin/convert.js
@@ -0,0 +1,449 @@
+var startTime = new Date().getTime();
+var fs = require("fs");
+var ueberDB = require("ueberDB");
+var mysql = require("mysql");
+var async = require("async");
+var Changeset = require("../node/utils/Changeset");
+var AttributePoolFactory = require("../node/utils/AttributePoolFactory");
+
+var settingsFile = process.argv[2];
+var sqlOutputFile = process.argv[3];
+
+//stop if the settings file is not set
+if(!settingsFile || !sqlOutputFile)
+{
+  console.error("Use: node convert.js $SETTINGSFILE $SQLOUTPUT");
+  process.exit(1);
+}
+
+log("read settings file...");
+//read the settings file and parse the json
+var settings = JSON.parse(fs.readFileSync(settingsFile, "utf8"));
+log("done");
+
+log("open output file...");
+var sqlOutput = fs.openSync(sqlOutputFile, "w");
+var sql = "SET CHARACTER SET UTF8;\n" +
+          "CREATE TABLE IF NOT EXISTS `store` ( \n" +
+          "`key` VARCHAR( 100 ) NOT NULL , \n" + 
+          "`value` LONGTEXT NOT NULL , \n" + 
+          "PRIMARY KEY (  `key` ) \n" +
+          ") ENGINE = INNODB;\n" + 
+          "START TRANSACTION;\n\n";
+fs.writeSync(sqlOutput, sql);
+log("done");
+
+//set setings for ep db
+var etherpadDB= new mysql.Client();
+etherpadDB.host = settings.etherpadDB.host; 
+etherpadDB.port = settings.etherpadDB.port;
+etherpadDB.database = settings.etherpadDB.database; 
+etherpadDB.user = settings.etherpadDB.user; 
+etherpadDB.password = settings.etherpadDB.password; 
+
+//get the timestamp once
+var timestamp = new Date().getTime();
+
+var padIDs;
+
+async.series([
+  //get all padids out of the database...
+  function(callback)
+  {
+    log("get all padIds out of the database...");
+    
+    etherpadDB.query("SELECT ID FROM PAD_META LIMIT", [], function(err, _padIDs)
+    {
+      padIDs = _padIDs;
+      callback(err);
+    });
+  },
+  function(callback)
+  {
+    log("done");
+    
+    //create a queue with a concurrency 100
+    var queue = async.queue(function (padId, callback) 
+    {
+      convertPad(padId, function(err)
+      {
+        incrementPadStats();
+        callback(err);
+      });
+    }, 100);
+    
+    //set the step callback as the queue callback
+    queue.drain = callback;
+    
+    //add the padids to the worker queue
+    for(var i=0,length=padIDs.length;i<length;i++)
+    {
+      queue.push(padIDs[i].ID);
+    }
+  }
+], function(err)
+{
+  if(err) throw err;
+  
+  //write the groups
+  var sql = "";
+  for(var proID in proID2groupID)
+  {
+    var groupID = proID2groupID[proID];
+    var subdomain = proID2subdomain[proID];
+    
+    sql+="REPLACE INTO store VALUES (" + etherpadDB.escape("group:" + groupID) + ", " + etherpadDB.escape(JSON.stringify(groups[groupID]))+ ");\n";
+    sql+="REPLACE INTO store VALUES (" + etherpadDB.escape("mapper2group:subdomain:" + subdomain) + ", " + etherpadDB.escape(groupID)+ ");\n";
+  }
+  
+  //close transaction
+  sql+="COMMIT;";
+  
+  //end the sql file
+  fs.writeSync(sqlOutput, sql, undefined, "utf-8");
+  fs.closeSync(sqlOutput);
+  
+  log("finished.");
+  process.exit(0);
+});
+
+function log(str)
+{
+  console.log((new Date().getTime() - startTime)/1000 + "\t" + str);
+}
+
+var padsDone = 0;
+
+function incrementPadStats()
+{
+  padsDone++;
+  
+  if(padsDone%100 == 0)
+  {
+    var averageTime = Math.round(padsDone/((new Date().getTime() - startTime)/1000));
+    log(padsDone + "/" + padIDs.length + "\t" + averageTime + " pad/s")
+  }
+}
+
+var proID2groupID = {};
+var proID2subdomain = {};
+var groups = {};
+
+function convertPad(padId, callback)
+{
+  var changesets = [];
+  var changesetsMeta = [];
+  var chatMessages = [];
+  var authors = [];
+  var apool;
+  var subdomain;
+  var padmeta;
+
+  async.series([
+    //get all needed db values
+    function(callback)
+    {
+      async.parallel([
+        //get the pad revisions
+        function(callback)
+        {
+          var sql = "SELECT * FROM `PAD_CHAT_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_CHAT_META` WHERE ID=?)";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err) 
+            {
+              //parse the pages
+              for(var i=0,length=results.length;i<length;i++)
+              {
+                parsePage(chatMessages, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
+              }
+            }
+            
+            callback(err);
+          });
+        },
+        //get the chat entries
+        function(callback)
+        {
+          var sql = "SELECT * FROM `PAD_REVS_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_REVS_META` WHERE ID=?)";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err) 
+            {
+              //parse the pages
+              for(var i=0,length=results.length;i<length;i++)
+              {
+                parsePage(changesets, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, false);
+              }
+            }
+            
+            callback(err);
+          });
+        },
+        //get the pad revisions meta data
+        function(callback)
+        {
+          var sql = "SELECT * FROM `PAD_REVMETA_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_REVMETA_META` WHERE ID=?)";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err) 
+            {
+              //parse the pages
+              for(var i=0,length=results.length;i<length;i++)
+              {
+                parsePage(changesetsMeta, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
+              }
+            }
+            
+            callback(err);
+          });
+        },
+        //get the attribute pool of this pad
+        function(callback)
+        {
+          var sql = "SELECT `JSON` FROM `PAD_APOOL` WHERE `ID` = ?";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err)
+            {
+              apool=JSON.parse(results[0].JSON).x;
+            }
+            
+            callback(err);
+          });
+        },
+        //get the authors informations
+        function(callback)
+        {
+          var sql = "SELECT * FROM `PAD_AUTHORS_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_AUTHORS_META` WHERE ID=?)";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err) 
+            {
+              //parse the pages
+              for(var i=0, length=results.length;i<length;i++)
+              {
+                parsePage(authors, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
+              }
+            }
+            
+            callback(err);
+          });
+        },
+        //get the pad information
+        function(callback)
+        {
+          var sql = "SELECT JSON FROM `PAD_META` WHERE ID=?";
+          
+          etherpadDB.query(sql, [padId], function(err, results)
+          {
+            if(!err) 
+            {
+              padmeta = JSON.parse(results[0].JSON).x;
+            }
+            
+            callback(err);
+          });
+        },
+        //get the subdomain
+        function(callback)
+        {
+          //skip if this is no proPad
+          if(padId.indexOf("$") == -1)
+          {
+            callback();
+            return;
+          }
+          
+          //get the proID out of this padID
+          var proID = padId.split("$")[0];
+        
+          var sql = "SELECT subDomain FROM pro_domains WHERE ID = ?";
+          
+          etherpadDB.query(sql, [proID], function(err, results)
+          {
+            if(!err)
+            {
+              subdomain = results[0].subDomain;
+            }
+            
+            callback(err);
+          });
+        }
+      ], callback);
+    },
+    function(callback)
+    {
+      //saves all values that should be written to the database
+      var values = {};
+      
+      //this is a pro pad, let's convert it to a group pad
+      if(padId.indexOf("$") != -1)
+      {
+        var padIdParts = padId.split("$");
+        var proID = padIdParts[0];
+        var padName = padIdParts[1];
+        
+        var groupID
+        
+        //this proID is not converted so far, do it
+        if(proID2groupID[proID] == null)
+        {
+          groupID = "g." + randomString(16);
+          
+          //create the mappers for this new group
+          proID2groupID[proID] = groupID;
+          proID2subdomain[proID] = subdomain;
+          groups[groupID] = {pads: {}};
+        }
+        
+        //use the generated groupID;
+        groupID = proID2groupID[proID];
+        
+        //rename the pad
+        padId = groupID + "$" + padName;
+        
+        //set the value for this pad in the group
+        groups[groupID].pads[padId] = 1;
+      }
+      
+      try
+      {
+        var newAuthorIDs = {};
+        var oldName2newName = {};
+        
+        //replace the authors with generated authors
+        // we need to do that cause etherpad saves pad local authors, etherpad lite uses them global
+        for(var i in apool.numToAttrib)
+        {
+          var key = apool.numToAttrib[i][0];
+          var value = apool.numToAttrib[i][1];
+          
+          //skip non authors and anonymous authors
+          if(key != "author" || value == "")
+            continue;
+            
+          //generate new author values
+          var authorID = "a." + randomString(16);
+          var authorColorID = authors[i].colorId || Math.floor(Math.random()*32);
+          var authorName = authors[i].name || null;
+          
+          //overwrite the authorID of the attribute pool
+          apool.numToAttrib[i][1] = authorID;
+          
+          //write the author to the database
+          values["globalAuthor:" + authorID] = {"colorId" : authorColorID, "name": authorName, "timestamp": timestamp};
+          
+          //save in mappers
+          newAuthorIDs[i] = authorID;
+          oldName2newName[value] = authorID;
+        }
+        
+        //save all revisions
+        for(var i=0;i<changesets.length;i++)
+        {
+          values["pad:" + padId + ":revs:" + i] = {changeset: changesets[i], 
+                                                   meta : {
+                                                     author: newAuthorIDs[changesetsMeta[i].a],
+                                                     timestamp: changesetsMeta[i].t,
+                                                     atext: changesetsMeta[i].atext || undefined
+                                                   }};
+        }
+        
+        //save all chat messages
+        for(var i=0;i<chatMessages.length;i++)
+        {
+          values["pad:" + padId + ":chat:" + i] = {"text": chatMessages[i].lineText, 
+                                                   "userId": oldName2newName[chatMessages[i].userId], 
+                                                   "time": chatMessages[i].time}
+        }
+        
+        //generate the latest atext
+        var fullAPool = AttributePoolFactory.createAttributePool().fromJsonable(apool);
+        var keyRev = Math.floor(padmeta.head / padmeta.keyRevInterval) * padmeta.keyRevInterval;
+        var atext = changesetsMeta[keyRev].atext;
+        var curRev = keyRev;
+        while (curRev < padmeta.head) 
+        {
+          curRev++;
+          var changeset = changesets[curRev];
+          atext = Changeset.applyToAText(changeset, atext, fullAPool);
+        }
+        
+        values["pad:" + padId] = {atext: atext, 
+                                  pool: apool, 
+                                  head: padmeta.head, 
+                                  chatHead: padmeta.numChatMessages }
+      
+      }
+      catch(e)
+      {
+        console.error("Error while converting pad " + padId + ", pad skiped");
+        console.error(e.stack ? e.stack : JSON.stringify(e));
+        callback();
+        return;
+      }
+      
+      var sql = "";
+      for(var key in values)
+      {
+        sql+="REPLACE INTO store VALUES (" + etherpadDB.escape(key) + ", " + etherpadDB.escape(JSON.stringify(values[key]))+ ");\n";
+      }
+      
+      fs.writeSync(sqlOutput, sql, undefined, "utf-8");
+      callback();
+    }
+  ], callback);
+}
+
+/**
+ * This parses a Page like Etherpad uses them in the databases
+ * The offsets descripes the length of a unit in the page, the data are
+ * all values behind each other
+ */
+function parsePage(array, pageStart, offsets, data, json)
+{
+  var start = 0;
+  var lengths = offsets.split(",");
+  
+  for(var i=0;i<lengths.length;i++)
+  {
+    var unitLength = lengths[i];
+    
+    //skip empty units
+    if(unitLength == "")
+      continue;
+    
+    //parse the number
+    unitLength = Number(unitLength);
+      
+    //cut the unit out of data
+    var unit = data.substr(start, unitLength);
+    
+    //put it into the array
+    array[pageStart + i] = json ? JSON.parse(unit) : unit;
+    
+    //update start
+    start+=unitLength;
+  }
+}
+
+/**
+ * Generates a random String with the given length. Is needed to generate the Author Ids
+ */
+function randomString(len) 
+{
+  var chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+  var randomstring = '';
+  for (var i = 0; i < len; i++)
+  {
+    var rnum = Math.floor(Math.random() * chars.length);
+    randomstring += chars.substring(rnum, rnum + 1);
+  }
+  return randomstring;
+}
diff --git a/bin/convertSettings.json.template b/bin/convertSettings.json.template
new file mode 100644
index 00000000..9115a4cd
--- /dev/null
+++ b/bin/convertSettings.json.template
@@ -0,0 +1,21 @@
+{
+  "etherpad-db": 
+  {
+    "host": "localhost",
+    "port": 3306,
+    "database": "etherpad",
+    "user": "etherpaduser",
+    "password": "yourpassword"
+  },
+  "etherpad-lite-db":
+  {
+    "dbType" : "mysql",
+    "dbSettings" : 
+    {
+      "user"    : "root", 
+      "host"    : "localhost", 
+      "password": "", 
+      "database": "store"
+    }
+  }
+}