Geek Answers Handbook

Fast natural sorting for large XML file in browser?

I have a problem right now that is the result of current restrictions on a server that our team does not control.

We have work to do using the database, but we are forced to use an XML file and parse it using Javascript / jQuery. We don’t even have write access for our scripts (only through our FTP account) ... we don’t like talking about it, but this is what we got.

The problem with these restrictions is that we need to parse a large XML file of about 500 kb in size with 1700-ary records of document name / document number / url.

The number is rather complicated, for example, "31-2b-1029E" mixed with a material of the type "T2315342".

So, I realized that I need to use something called "Natural Sort" (thanks stackoverflow).

Anyway, I tried using this script here:

/*
 * Reference: http://www.overset.com/2008/09/01/javascript-natural-sort-algorithm/
 * Natural Sort algorithm for Javascript - Version 0.6 - Released under MIT license
 * Author: Jim Palmer (based on chunking idea from Dave Koelle)
 * Contributors: Mike Grier (mgrier.com), Clint Priest, Kyle Adams, guillermo
 */
function naturalSort (a, b) {
    var re = /(^-?[0-9]+(\.?[0-9]*)[df]?e?[0-9]?$|^0x[0-9a-f]+$|[0-9]+)/gi,
        sre = /(^[ ]*|[ ]*$)/g,
        dre = /(^([\w ]+,?[\w ]+)?[\w ]+,?[\w ]+\d+:\d+(:\d+)?[\w ]?|^\d{1,4}[\/\-]\d{1,4}[\/\-]\d{1,4}|^\w+, \w+ \d+, \d{4})/,
        hre = /^0x[0-9a-f]+$/i,
        ore = /^0/,
        // convert all to strings and trim()
        x = a.toString().replace(sre, '') || '',
        y = b.toString().replace(sre, '') || '',
        // chunk/tokenize
        xN = x.replace(re, '\0$1\0').replace(/\0$/,'').replace(/^\0/,'').split('\0'),
        yN = y.replace(re, '\0$1\0').replace(/\0$/,'').replace(/^\0/,'').split('\0'),
        // numeric, hex or date detection
        xD = parseInt(x.match(hre)) || (xN.length != 1 && x.match(dre) && Date.parse(x)),
        yD = parseInt(y.match(hre)) || xD && y.match(dre) && Date.parse(y) || null;
    // first try and sort Hex codes or Dates
    if (yD)
        if ( xD < yD ) return -1;
        else if ( xD > yD ) return 1;
    // natural sorting through split numeric strings and default strings
    for(var cLoc=0, numS=Math.max(xN.length, yN.length); cLoc < numS; cLoc++) {
        // find floats not starting with '0', string or 0 if not defined (Clint Priest)
        oFxNcL = !(xN[cLoc] || '').match(ore) && parseFloat(xN[cLoc]) || xN[cLoc] || 0;
        oFyNcL = !(yN[cLoc] || '').match(ore) && parseFloat(yN[cLoc]) || yN[cLoc] || 0;
        // handle numeric vs string comparison - number < string - (Kyle Adams)
        if (isNaN(oFxNcL) !== isNaN(oFyNcL)) return (isNaN(oFxNcL)) ? 1 : -1; 
        // rely on string comparison if different types - i.e. '02' < 2 != '02' < '2'
        else if (typeof oFxNcL !== typeof oFyNcL) {
            oFxNcL += ''; 
            oFyNcL += ''; 
        }
        if (oFxNcL < oFyNcL) return -1;
        if (oFxNcL > oFyNcL) return 1;
    }
    return 0;
}

And applied using:

// Natural Sort (disabled because it is super freaking slow.... need xsl transform sorting instead)
var sortedSet = $(data).children("documents").children("document").sort(function(a, b) {
    return naturalSort($(a).children('index').text(), $(b).children('index').text());
});

This works fine in our other, smaller XML file, but for the giant 500kb-ish file Safari (v4) just hangs for a few minutes to sort it, and Firefox (the last one) takes about 10 seconds to process (still not good, but at least reasonable).

I also found this other smaller / lighter script called Alphanum :

function alphanum(a, b) {
  function chunkify(t) {
    var tz = [], x = 0, y = -1, n = 0, i, j;

    while (i = (j = t.charAt(x++)).charCodeAt(0)) {
      var m = (i == 46 || (i >=48 && i <= 57));
      if (m !== n) {
        tz[++y] = "";
        n = m;
      }
      tz[y] += j;
    }
    return tz;
  }

  var aa = chunkify(a);
  var bb = chunkify(b);

  for (x = 0; aa[x] && bb[x]; x++) {
    if (aa[x] !== bb[x]) {
      var c = Number(aa[x]), d = Number(bb[x]);
      if (c == aa[x] && d == bb[x]) {
        return c - d;
      } else return (aa[x] > bb[x]) ? 1 : -1;
    }
  }
  return aa.length - bb.length;
}

This speeds up for Safari, but still blocks the browser for a minute or so.

, , XSL XML, , -, - , JavaScript.

-, : Sarissa, , , 2011-06-22.

, xslt.js

:

  • XSL ?
  • , XSL ? (URL- ?)
  • , ?
  • XSL , ?

, .

+5
3

, +1.

XSLT 1.0 ( XSLT 2.0, , , , 5 XSLT 2.0 ):

<xsl:stylesheet version="1.0"
 xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 xmlns:ext="http://exslt.org/common" exclude-result-prefixes="xml">
 <xsl:output omit-xml-declaration="yes" indent="yes"/>
 <xsl:strip-space elements="*"/>

 <xsl:variable name="vDigits" select="'0123456789'"/>

 <xsl:variable name="vPadding" select=
 "'                    '"/>

 <xsl:variable name="vMaxNumLength"
      select="string-length($vPadding)"/>

 <xsl:template match="/">
  <xsl:variable name="vrtfPass1">
   <t>
    <xsl:apply-templates/>
   </t>
  </xsl:variable>

  <xsl:variable name="vPass1" select="ext:node-set($vrtfPass1)"/>

  <t>
    <xsl:for-each select="$vPass1/*/*">
     <xsl:sort select="@sortMe"/>

     <xsl:copy>
      <xsl:value-of select="."/>
     </xsl:copy>
    </xsl:for-each>
  </t>
 </xsl:template>

 <xsl:template match="str">
   <str>
    <xsl:apply-templates select="text()" mode="normalize"/>
    <xsl:copy-of select="text()"/>
   </str>
 </xsl:template>

 <xsl:template match="text()" mode="normalize" name="normalize">
  <xsl:param name="pText" select="."/>
  <xsl:param name="pAccum" select="''"/>

  <xsl:choose>
   <xsl:when test="not(string-length($pText) >0)">
     <xsl:attribute name="sortMe">
       <xsl:value-of select="$pAccum"/>
     </xsl:attribute>
   </xsl:when>
   <xsl:otherwise>
    <xsl:variable name="vChar1" select="substring($pText,1,1)"/>

    <xsl:choose>
     <xsl:when test="not(contains($vDigits,$vChar1))">
       <xsl:variable name="vDig1" select=
       "substring(translate($pText,
                            translate($pText, $vDigits, ''),
                            ''
                            ),
                  1,1)"/>
       <xsl:variable name="vDig">
        <xsl:choose>
         <xsl:when test="string-length($vDig1)">
          <xsl:value-of select="$vDig1"/>
         </xsl:when>
         <xsl:otherwise>0</xsl:otherwise>
        </xsl:choose>
       </xsl:variable>

       <xsl:variable name="vNewText" select=
        "substring-before(concat($pText,$vDig), $vDig)"/>

       <xsl:call-template name="normalize">
        <xsl:with-param name="pText" select=
         "substring($pText, string-length($vNewText)+1)"/>
        <xsl:with-param name="pAccum" select=
        "concat($pAccum, $vNewText)"/>
       </xsl:call-template>
     </xsl:when>

     <xsl:otherwise>
      <xsl:variable name="vNonDig1" select=
      "substring(translate($pText, $vDigits, ''),1,1)"/>

      <xsl:variable name="vNonDig">
        <xsl:choose>
         <xsl:when test="string-length($vNonDig1)">
          <xsl:value-of select="$vNonDig1"/>
         </xsl:when>
         <xsl:otherwise>Z</xsl:otherwise>
        </xsl:choose>
      </xsl:variable>

      <xsl:variable name="vNum" select=
           "substring-before(concat($pText,'Z'),$vNonDig)"/>

      <xsl:variable name="vNumLength" select=
       "string-length($vNum)"/>

      <xsl:variable name="vNewText" select=
       "concat(substring($vPadding,
                         1,
                         $vMaxNumLength -$vNumLength),
               $vNum
               )"/>

       <xsl:call-template name="normalize">
        <xsl:with-param name="pText" select=
         "substring($pText, $vNumLength +1)"/>
        <xsl:with-param name="pAccum" select=
        "concat($pAccum, $vNewText)"/>
       </xsl:call-template>
     </xsl:otherwise>
    </xsl:choose>
   </xsl:otherwise>
  </xsl:choose>
 </xsl:template>
</xsl:stylesheet>

, XML- :

<t>
 <str>Allegia 6R Clasteron</str>
 <str>200X Radonius</str>
 <str>Xiph Xlater 10000</str>
 <str>1000X Radonius Maximus</str>
 <str>Callisto Morphamax 6000 SE</str>
 <str>10X Radonius</str>
 <str>20X Radonius</str>
 <str>30X Radonius</str>
 <str>20X Radonius Prime</str>
 <str>40X Radonius</str>
 <str>Allegia 50 Clasteron</str>
 <str>Allegia 500 Clasteron</str>
 <str>Allegia 50B Clasteron</str>
 <str>Allegia 51 Clasteron</str>
 <str>Alpha 100</str>
 <str>Alpha 2</str>
 <str>Alpha 200</str>
 <str>Alpha 2A</str>
 <str>Alpha 2A-8000</str>
 <str>Alpha 2A-900</str>
 <str>Callisto Morphamax</str>
 <str>Callisto Morphamax 500</str>
 <str>Callisto Morphamax 5000</str>
 <str>Callisto Morphamax 600</str>
 <str>Callisto Morphamax 6000 SE2</str>
 <str>Callisto Morphamax 700</str>
 <str>Callisto Morphamax 7000</str>
 <str>Xiph Xlater 2000</str>
 <str>Xiph Xlater 300</str>
 <str>Xiph Xlater 40</str>
 <str>Xiph Xlater 5</str>
 <str>Xiph Xlater 50</str>
 <str>Xiph Xlater 500</str>
 <str>Xiph Xlater 5000</str>
 <str>Xiph Xlater 58</str>
</t>

, "" :

<t>
   <str>10X Radonius</str>
   <str>20X Radonius</str>
   <str>20X Radonius Prime</str>
   <str>30X Radonius</str>
   <str>40X Radonius</str>
   <str>200X Radonius</str>
   <str>1000X Radonius Maximus</str>
   <str>Allegia 6R Clasteron</str>
   <str>Allegia 50 Clasteron</str>
   <str>Allegia 50B Clasteron</str>
   <str>Allegia 51 Clasteron</str>
   <str>Allegia 500 Clasteron</str>
   <str>Alpha 2</str>
   <str>Alpha 2A</str>
   <str>Alpha 2A-900</str>
   <str>Alpha 2A-8000</str>
   <str>Alpha 100</str>
   <str>Alpha 200</str>
   <str>Callisto Morphamax</str>
   <str>Callisto Morphamax 500</str>
   <str>Callisto Morphamax 600</str>
   <str>Callisto Morphamax 700</str>
   <str>Callisto Morphamax 5000</str>
   <str>Callisto Morphamax 6000 SE</str>
   <str>Callisto Morphamax 6000 SE2</str>
   <str>Callisto Morphamax 7000</str>
   <str>Xiph Xlater 5</str>
   <str>Xiph Xlater 40</str>
   <str>Xiph Xlater 50</str>
   <str>Xiph Xlater 58</str>
   <str>Xiph Xlater 300</str>
   <str>Xiph Xlater 500</str>
   <str>Xiph Xlater 2000</str>
   <str>Xiph Xlater 5000</str>
   <str>Xiph Xlater 10000</str>
</t>

. , 40 . , , , , / .

, :

XML-, , 1700 str 0.659 . 8- Pentium Pentium, 2 .

  • .

  • " ", , sortMe str. text-node str - 40.

  • Pass 2 str - sortMe.

, 4 :

:

XSL ?
, XSL ? ( ?)
, , ?
XSL , ?

  • ( ) . XSLT .

  • XSLT "" .

  • - . , PL, .

  • PL, XSLT, .

+5

:

(a) Sarissa XSLT-, Javascript, API Javascript XSLT-, .

(b) xslt.js - , XSLT- Javascript. , .

- Saxon-CE, - ( Java Javascript GWT). XSLT 2.0 . Saxon , " " (<xsl:sort collation='http://saxon.sf.net/collation?alphanumeric=yes'/>), Saxon-CE.

(P.S. " ". .)

+1

, , . 1700 func, , 10 000 750 000 ... , , , .

, , jquery . . , , . xml, jquery , , . . xml.

0

More articles:


All Articles