BigInt.js: http://leemon.com/crypto/BigInt.html

Функции для декодирования: hex: http://farhadi.ir/works/hex Base64: http://farhadi.ir/works/base64

var test_key = "81DA1AD21759F9B8160F737E717482C56AB7BBCA2A9280B223FD8283C5CCD286E7CAF4F765954B0B4BD0746DB040CD226E444E50567AF6563F964006A9A81D9F#10001";
var test_toenc = '6dc8373faa9a424988341d3f25ffeaec';



var keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";

/* Base64 conversion methods.
 * Copyright (c) 2006 by Ali Farhadi.
 * released under the terms of the Gnu Public License.
 * see the GPL for details.
 *
 * Email: ali[at]farhadi[dot]ir
 * Website: http://farhadi.ir/
 */

//Encodes data to Base64 format
function base64Encode(data){
	if (typeof(btoa) == 'function') return btoa(data);//use internal base64 functions if available (gecko only)
	var b64_map = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
	var byte1, byte2, byte3;
	var ch1, ch2, ch3, ch4;
	var result = new Array(); //array is used instead of string because in most of browsers working with large arrays is faster than working with large strings
	var j=0;
	for (var i=0; i<data.length; i+=3) {
		byte1 = data.charCodeAt(i);
		byte2 = data.charCodeAt(i+1);
		byte3 = data.charCodeAt(i+2);
		ch1 = byte1 >> 2;
		ch2 = ((byte1 & 3) << 4) | (byte2 >> 4);
		ch3 = ((byte2 & 15) << 2) | (byte3 >> 6);
		ch4 = byte3 & 63;
		
		if (isNaN(byte2)) {
			ch3 = ch4 = 64;
		} else if (isNaN(byte3)) {
			ch4 = 64;
		}

		result[j++] = b64_map.charAt(ch1)+b64_map.charAt(ch2)+b64_map.charAt(ch3)+b64_map.charAt(ch4);
	}

	return result.join('');
}


/* Hexadecimal conversion methods.
 * Copyright (c) 2006 by Ali Farhadi.
 * released under the terms of the Gnu Public License.
 * see the GPL for details.
 *
 * Email: ali[at]farhadi[dot]ir
 * Website: http://farhadi.ir/
 */

//Encodes data to Hex(base16) format
function hexEncode(data){
	var b16_digits = '0123456789abcdef';
	var b16_map = new Array();
	for (var i=0; i<256; i++) {
		b16_map[i] = b16_digits.charAt(i >> 4) + b16_digits.charAt(i & 15);
	}
	
	var result = new Array();
	for (var i=0; i<data.length; i++) {
		result[i] = b16_map[data.charCodeAt(i)];
	}
	
	return result.join('');
}

//Decodes Hex(base16) formated data
function hexDecode(data){
	var b16_digits = '0123456789abcdef';
	var b16_map = new Array();
	for (var i=0; i<256; i++) {
		b16_map[b16_digits.charAt(i >> 4) + b16_digits.charAt(i & 15)] = String.fromCharCode(i);
	}
	if (!data.match(/^[a-f0-9]*$/i)) return false;// return false if input data is not a valid Hex string
	
	if (data.length % 2) data = '0'+data;
		
	var result = new Array();
	var j=0;
	for (var i=0; i<data.length; i+=2) {
		result[j++] = b16_map[data.substr(i,2)];
	}

	return result.join('');
}

function ord( string ) {	// Return ASCII value of character
	code = string.charCodeAt(0);
	if(code>900)code=code-848; // Может и не 900, но так работает точно. 
	return code;
}

function encrypt_yarsa(key, toenc)
{

	var toenc_array = new Array();
	var DATA_ARR = new Array();
	var ord_str='';
	var toenc_str='';
	for(i=0;i<toenc.length;i++){
		toenc_array[i] = toenc.substr(i,1); //Массив с символами
		DATA_ARR[i] = ord(toenc_array[i]); //строка с ord
		ord_str+=','+ ord(toenc_array[i]); //Строка для дебага
	}
	var NSTR = key.split('#')[0];
	var ESTR = key.split('#')[1];
	var N = new String(  bigInt2str(str2bigInt(NSTR,  16,0),10)  );
	var E = new String(  bigInt2str(str2bigInt(ESTR,  16,0),10)  );
	var STEP_SIZE = NSTR.length/2-1;

	//alert('NSTR:'+NSTR + '\nESTR:' + ESTR + '\nN:'+N+'\nE:'+E+'\nStep:'+STEP_SIZE);

	var prev_crypted = new Array(24);

	var hex_out= new String("");

	for(i=0; i<((DATA_ARR.length-1)/(STEP_SIZE+1));i++)
	{
		tmp = DATA_ARR.slice(i*STEP_SIZE, (i+1)*STEP_SIZE);
		tmp_str='';
		for(j=0;j<tmp.length;j++)
		{
			tmp[j] = (tmp[j] ^ prev_crypted[j]);
			tmp_str += ','+tmp[j];
		}
		tmp.reverse();  //#29
		var plain_str = new String('');
		plain = str2bigInt(plain_str,10,0);
		for(x=0 ; x<tmp.length;x++) //#31
		{
			currBase=10;
			var m1 = new String('256');
			var m2 = new String(x);
			var n1=str2bigInt(m1,currBase,0);
			var n2=str2bigInt(m2,currBase,0);  
			var n3=str2bigInt(N,currBase,0);
			pow1 = powMod(n1, n2, n3);
			tmp_x = new String(tmp[x]);
			tmp_x_big = str2bigInt(tmp_x,currBase,0);
			pow2 = mult(pow1, tmp_x_big);
			plain = add(plain, pow2);
		}

		var E1=str2bigInt(E,10,0);
		var N1=str2bigInt(N,10,0);
		plain_pow = powMod(plain, E1, N1);
		plain_pow_str = bigInt2str(plain_pow, 16);

		//Аналог: hex_result = "".join(['0']*( len(NSTR)- len(hex_result))) + hex_result
		hex_result = new Array((NSTR.length - plain_pow_str.length) + 1).join('0') + plain_pow_str;

		min_x = Math.min(hex_result.length, prev_crypted.length*2);
		//alert('PLAIN:' + bigInt2str(plain,10) + '\n' + 	'plain_pow:' +  plain_pow_str + '\n' + 'Min_x:'+min_x);

		prev_crypted_tmp='';
		for(x=0;x<min_x;x=x+2)//Через 2
		{
			prev_crypted[x/2] = parseInt("0x"+hex_result.substr(x, 2));
			prev_crypted_tmp+=','+prev_crypted[x/2];
		}
		//alert('prev_crypted_tmp'+prev_crypted_tmp);

		//Аналог: ("0" if len(tmp) < 16 else "")
		if(tmp.length < 16) hex_out+="00";
		
		//+ ("%x" % len(tmp)) + "00" # current size
		hex_out += tmp.length.toString(16).toUpperCase() + "00";
		//ks = len(NSTR)/2
		ks = NSTR.length/2;
		//Аналог: hex_out += ("0" if ks < 16 else "") + ("%x" % ks) + "00" # key size
		if(ks<16) hex_out += "0";
		hex_out += ks.toString(16).toUpperCase() + "00"; 
		hex_out += hex_result;
		//document.write('HEX_OUT' + hex_out+"<br>");

	}
	return base64Encode(hexDecode(hex_out.toLowerCase())).replace(/[\n\r\t]/g, "");
}
my_result = encrypt_yarsa(test_key, test_toenc);
document.write('<b>' + my_result + "</b><br>");