How we test the ACF plugin for each new release
Testing the ACF plugin for each new release is a comprehensive work to ensure everything is functioning as expected.
There is the following categories of testing we run through.
- Automated tests for testing basic functionality.
- Testing with all ACF functions we have ever developed, to see that it produce the desired result.
- Stress testing, field calculations using ACF functions, shown in portals, thousands of calculations in short time. Doing export of those calculated fields in large databases. Running ACF functions pulling calculated fields in SQL queries.
- Testing on Server as PSOS scripts vs Client - See that they both works as expected.
New versions of released plugins follow this path. For releases on new plattforms, also some long running tests are needed with multiple users to ensure everything is correct.
How do we do Automated tests.
Testing basic functionality in the ACF language, requres us to run specialized test functions that run a lot of simple functions and compare with a known result. We have a test set of functions that is always run before release:
- Test Math functions
- Test String functions
- Test Cryptographic functions
- Test File-IO / Directory and ZIP file functions.
- Testing SQL functions
- Test Excel production - From Demo databases.
- Testing API functions.
Here are some examples:
First we have some "Assert" functions, that compare the two first parameters, and throw an exception if they are different. We have one such for each datatype we compare:
// Strings
function assert ( string a1, string a2, string message )
if ( a1 != a2 ) then
throw "Error: " + format("%s, s1='%s', s2='%s'", message, a1, a2);
end if
return true;
end
// Floating numbers
function assertF ( float f1, float f2, string message )
if ( round(f1,5) != round(f2,5)) then
throw "Error: " + message + format (", f1=%f, f2=%f", f1, f2);
end if
return true;
end
// Long numbers (Can be used from its too)
function assertL ( long L1, long L2, string message )
if ( L1 != L2) then
throw "Error: " + message + format (", L1=%ld, L2=%ld", l1, l2);
end if
return true;
end
// Boolean values
function assertB ( bool L1, bool L2, string message )
if ( L1 != L2) then
throw "Error: " + message + format (", L1=%d, L2=%d", int(l1), int(l2));
end if
return true;
end
Then a test for Math functions looks like this:
/*
AnnuityLoanPayment:
Calculate the Payment amounth for an annuity loan :
PV = Present Value
r = Interest rate
n = number of payments
*/
function AnnuityLoanPayment ( float PV, float r, int n)
float P = r*PV/(1-(1+r)^(-n));
return P;
end
/*
CalcAnnuityInterestRate:
Calculate the Interest rate for an annuity loan by simulation :
LoanSum = Present Value
P = Payment amounth
Y = number of years
nY = number of payments pr year.
*/
function CalcAnnuityInterestRate ( float LoanSum, float P, int Y, int nY)
float r;
float res;
// We start with High Interest rate.
float rY = 100.0;
float step = rY/2;
float usedrY;
if (P*Y*nY < LoanSum) then
throw "\nNot enough payment - Payment starts at : " + LoanSum / (nY*Y);
else
repeat
usedrY = rY;
r = rY/100/nY;
res = AnnuityLoanPayment ( LoanSum, r, Y*nY);
print "\nInterest: " + rY + "% - Payment: " + res;
if ((res-P)>0.0) then
print " diff(+) " + (res-P);
rY = rY - step;
else
print " diff(-) " + (res-P);
rY = rY + step;
end if
step = step / 2;
until ((abs(res-P)<0.0001) || (step < 0.000001));
end if
return usedrY;
end
Then simple math, including the Annuity test above.
function test_math ()
bool a;
try
a = assertF(2*3+4*5, 26, "Priority of multiply versus addition failed");
a = assertF(10/2-3, 2, "Division and subtraction priority failed");
a = assertF(1.123456 + 2.234567, 3.35802, "Float addition rounding failed");
a = assertF((5.5 * 4.1) / 2.3 + 1.2, 11.004347826086954, "Complex formula test failed");
a = assertF(CalcAnnuityInterestRate(100000, 2000, 5, 12) ,7.4200958013534545898, "Annuity calc test failed");
a = assertF(cos(degreesToRadians(60)), 0.5, "Cosine of 60 degrees failed");
a = assertF(sin(degreesToRadians(30)), 0.5, "Sinus of 30 degrees failed");
a = assertF(tan(degreesToRadians(45)), 1, "Tangens of 45 degrees failed");
a = assertF(radiansToDegrees(acos(0.5)), 60, "Arccos of 0.5 failed");
a = assertF(radiansToDegrees(asin(0.5)), 30, "ArcSin of 0.5 failed");
a = assertF(radiansToDegrees(atan(1)), 45, "ArcTan of 1 failed");
a = assertF(sqrt(16), 4, "SQRT failed");
a = assertF(mod(10,7), 3, "Modulo failed");
a = assertF(fact(5), 120, "Faculty failed");
a = assertF(round(3.141592654, 3), 3.142, "Round failed");
catch
alert ( "Error: " + last_error);
a = false;
end try
return a;
end
Testing Strings:
function trimUserInput ( string t )
FunctionID 507;
t = trimboth (t);
// only lowercase (lacy user input)
if ( lower(t) == t) then
t = proper(t);
// Last word....: ".*\b(\w+)\b[.?!]?\s*$"
end if
// Remove double spaces, do it several times in case of many spaces.
t = substitute (t, " ", " ");
t = substitute (t, " ", " ");
t = substitute (t, " ", " ");
t = substitute (t, " ", " ");
return t;
end
function testCase(string val)
string bb;
case
:(val=="tt")
bb = "ttx";
:(val=="tx")
bb = "txx";
default
// in FileMaker: if (val = cc; "ccxx"; "deftt");
bb = (val == "cc" ? "ccxx":"deftt")*3;
end case
int i, j=10;
for (i=1,j)
if (i==3) then
break;
end if
end for
bb = bb+string(i);
return bb;
end
function test_strings ()
bool a;
int i;
array string arr1 = {"a", "b", "c"}, arr2;
set_locale ("nb_NO.UTF-8"); // We test with Norwegian Locale.
date d1, d2;
try
a = assert(left("abcdef", 3), "abc", "function 'left' failed");
a = assert(right("abcdef", 3), "def", "function 'right' failed");
a = assert(trimleft(" abcdef "), "abcdef ", "function 'trimleft' failed");
a = assert(trimright(" abcdef "), " abcdef", "function 'trimleft' failed");
a = assert(trimboth("\r abcdef \n"), "abcdef", "function 'trimboth' failed");
a = assert(upper("abcdefghijklmnopq"), "ABCDEFGHIJKLMNOPQ", "function 'upper' failed");
a = assert(upper("abcdefghijklmnopqæøå"), "ABCDEFGHIJKLMNOPQÆØÅ", "function 'upper' failed with norwegian æøå");
a = assert(lower("ABCDEFGHIJKLMNOPQ"), "abcdefghijklmnopq", "function 'lower' failed");
a = assert(proper("ABCDEFGH IJKLMNOPQ ola normann"), "Abcdefgh Ijklmnopq Ola Normann", "function 'proper' failed");
a = assert(proper("øst politidistrikt"), "Øst Politidistrikt", "function 'proper' failed on unicode chars");
a = assert(trimUserInput(" Øst Politidistrikt "), "Øst Politidistrikt", "Calling 'TrimUserInput' failed on unicode chars");
a = assertL(pos("abcdefghijklmnopq","def"), 3, "Pos function failed"); // returns 0-based index
a = assert(substitute("abcdefghijklmnopq", "ghi", "001"), "abcdef001jklmnopq", "Substitute failed");
a = assert(mid("abcdefghijklmnopq",3,2), "de", "Function mid failed");
a = assert(substring("abcdefghijklmnopq",3,2), "de", "Function substring failed");
a = assertL(length("abcdef"), 6, "Function length failed");
a = assertL(bytes("Bø"), 3, "Bytes on utf-8 string failed"); // 0x42 0xC3 0xB8
a = assertL(length("Bø"), 2, "Length on utf-8 string failed");
a = assertL(ascii("@"), 64, "Function ascii failed");
a = assert(char(65), "A", "Function char failed");
a = assertL(sizeof(arr1), 3, "Function sizeof failed");
a = assert(implode(";", arr1), "a;b;c", "Function implode failed");
arr2 = explode ( ";", "a;b;c");
a = assertL ( sizeof (arr2), 3, "Dimension after explode is wrong");
for (i= 1, 3)
assert (arr1[i], arr2[i], format ("%s, Index %d", "Explode failed", i));
end for
a = assert (between ( "[start]abc[end]", "[start]", "[end]"), "abc", "Between failed");
a = assertL(ValueCount ("a\rb\rc"), 3, "ValueCount failed");
d1 = date ( "2024-01-02", "%Y-%m-%d");
a = assert (string ( d1, "%d.%m.%Y"), "02.01.2024", "Date to string with format failed");
d2 = date ( "2024-01-10", "%Y-%m-%d");
a = assertL(d2-d1, 8, "Date difference failed");
a = assert (string ( d1+10, "%d.%m.%Y"), "12.01.2024", "Date addition failed");
a = assert ("=!"*3, "=!=!=!", "String multiply (repeat) failed");
a = assert ("bbb"+"AAA", "bbbAAA", "String Concatenation failed");
a = assert (testCase("tt"), "ttx3", "TestCase 1 Failed");
a = assert (testCase("tx"), "txx3", "TestCase 2 Failed");
a = assert (testCase("cc"), "ccxxccxxccxx3", "TestCase 3 Failed");
a = assert (testCase("ff"), "defttdefttdeftt3", "TestCase 4 Failed");
a = assertB(Min_(d1, d2) == !2024-01-02!,true , "Min_ dates failed");
a = assertB(Max_(d1, d2) == !2024-01-10!, true, "Max_ dates failed");
a = assertB ("Æ" < "Ø" && "Ø" < "Å", true, "Compare order unichar strings failed" );
a = assert (mdsnippet("**My Bold**\n\n_Some underline_ Text"), "<p><strong>My Bold</strong></p>\n\n<p><u>Some underline</u> Text</p>\n", "mdsnippet feiled");
catch
throw ( "Error: " + last_error);
a = false;
end try
return a;
end
Testing Cryptographic functions:
function test_hashFunctions ()
bool a;
string source = "Hello World!";
string key = "my_secret_key";
try
// MD5
a = assert(create_hash(md5, source, base64enc), "7Qdih1MuhjZehB6Sv8UNjA==", "MD5/Base64 failed");
a = assert(create_hash(md5, source, hexenc), "ed076287532e86365e841e92bfc50d8c", "MD5/Hex failed");
// SHA1
a = assert(create_hash(sha1, source, base64enc), "Lve95gjOVATpfV8EL5X4nxwjKHE=", "SHA1/Base64 failed");
a = assert(create_hash(sha1, source, hexenc), "2ef7bde608ce5404e97d5f042f95f89f1c232871", "SHA1/Hex failed");
// SHA256
a = assert(create_hash(sha256, source, base64enc), "f4OxZX/x/FO5LcGBSKHWXfwtSx+j1ncoSt3SABJtkGk=", "SHA256/Base64 failed");
a = assert(create_hash(sha256, source, hexenc), "7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "SHA256/Hex failed");
// SHA384
a = assert(create_hash(sha384, source, base64enc), "v9dsDrvQBv7lg0EFR8GIewKSvnbVgtlsJC0qeScj4/1v0GH51c/RO4+WE1jmrbpK", "SHA384/Base64 failed");
a = assert(create_hash(sha384, source, hexenc), "bfd76c0ebbd006fee583410547c1887b0292be76d582d96c242d2a792723e3fd6fd061f9d5cfd13b8f961358e6adba4a", "SHA384/Hex failed");
// SHA512
a = assert(create_hash(sha512, source, base64enc), "hhhE1nBOhXP+w02WfiC8/vPUJM9IvgTm3AjyvVjHKXQzcQFerYkcw88cnTS0kmS1EHUbH/nlN5N7xGtdb/TsyA==", "SHA512/Base64 failed");
a = assert(create_hash(sha512, source, hexenc), "861844d6704e8573fec34d967e20bcfef3d424cf48be04e6dc08f2bd58c729743371015ead891cc3cf1c9d34b49264b510751b1ff9e537937bc46b5d6ff4ecc8", "SHA512/Hex failed");
// HMAC MD5
a = assert(create_hmac(md5, key, source, base64enc), "AYH2AOvu2Cbt3Ef/hR9k7A==", "Failed HMAC MD5/base64");
a = assert(create_hmac(md5, key, source, hexenc), "0181f600ebeed826eddc47ff851f64ec", "Failed HMAC MD5/hex");
// HMAC SHA1
a = assert(create_hmac(sha1, key, source, base64enc), "BwpO356r2gx9SYj3UFVvc/BfdVA=", "Failed HMAC SHA1/base64");
a = assert(create_hmac(sha1, key, source, hexenc), "070a4edf9eabda0c7d4988f750556f73f05f7550", "Failed HMAC SHA1/hex");
// HMAC SHA256
a = assert(create_hmac(sha256, key, source, base64enc), "5z1prOAxmam9lz5nlHgudpxIhksTY8pClCivq7RUytQ=", "Failed HMAC SHA256/base64");
a = assert(create_hmac(sha256, key, source, hexenc), "e73d69ace03199a9bd973e6794782e769c48864b1363ca429428afabb454cad4", "Failed HMAC SHA256/hex");
// HMAC SHA384
a = assert(create_hmac(sha384, key, source, base64enc), "UDZdkjSVyMpFJQu9dmeAlKg+LW97o6yq1pfBj3QZJM4YMmgZxkoUVVUzyMDcP3DT", "Failed HMAC SHA384/base64");
a = assert(create_hmac(sha384, key, source, hexenc), "50365d923495c8ca45250bbd76678094a83e2d6f7ba3acaad697c18f741924ce18326819c64a14555533c8c0dc3f70d3", "Failed HMAC SHA384/hex");
// HMAC SHA512
a = assert(create_hmac(sha512, key, source, base64enc), "x38L+mMFPEM+QqGrPMOzx2l7rTlmNePxPdaYllFiLQm+G9K2Bf/9oQcci861HAF4NTNqsPZS8gPEiQ0KoznMlA==", "Failed HMAC SHA512/base64");
a = assert(create_hmac(sha512, key, source, hexenc), "c77f0bfa63053c433e42a1ab3cc3b3c7697bad396635e3f13dd6989651622d09be1bd2b605fffda1071c8bceb51c017835336ab0f652f203c4890d0aa339cc94", "Failed HMAC SHA512/hex");
catch
throw ( "Error: " + last_error);
a = false;
end try
return a;
end