Fixing and testing Fermat's factorization implementation.

XXX. Note that it is being currently tested against small numbers. It shall be
checked even with something more appropriate, like twin/sexy primes with a lot
of digits.

src/questions/fermat.c

@@ -41,22 +41,21 @@ fermat_question_ask(const RSA *rsa)
   BN_sqrtmod(tmp, rem, n, ctx);
   /* Δ = |p - q| = |a + b - a + b| = |2b| > √N  2⁻¹⁰⁰ */
   BN_rshift(dssdelta, tmp, 101);
-  /* a² = (⌊√N⌋ + 1)² =  N + 1 + 2⌊√N⌋ */
   BN_copy(a, tmp);
-  BN_uiadd1(a);
-  /* b² = a² - N */
-  BN_sub(b2, a2, n);
+  BN_sqr(a2, a, ctx);
 
   do {
-    /* b² += 2a + 1 */
+    /* a² += 2a + 1 */
     BN_lshift(tmp, a, 1);
     BN_uiadd1(tmp);
-    BN_uadd(b2, b2, tmp);
+    BN_uadd(a2, a2, tmp);
     /* a += 1 */
     BN_uiadd1(a);
+    /* b² = a² - N */
+    BN_usub(b2, a2, n);
     /* b */
     BN_sqrtmod(b, rem, b2, ctx);
-  } while (!BN_is_zero(rem) && BN_ucmp(b, dssdelta) == 1);
+  } while (!BN_is_zero(rem) && BN_ucmp(b, dssdelta) < 1);
 
   if (BN_is_zero(rem)) {
     /* p, q found :) */

src/questions/tests/Makefile.am

@@ -1,9 +1,8 @@
 # unittesting my ass
 LDADD=../libquestions.a -lssl -lcrypto
 
-TESTS = test_qarith test_qstrings test_wiener test_pollard
-check_PROGRAMS = $(TESTS)
-# check_LIBRARIES = libquestions.a
+check_PROGRAMS = test_qarith test_qstrings test_wiener test_pollard
+TESTS = $(check_PROGRAMS) test_fermat.test
 
 test_qstrings_SOURCES = test_qstrings.c
 test_qarith_SOURCES = test_qarith.c

src/questions/tests/fermat.pem

@@ -0,0 +1,5 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIGGAoGALJvzyef52TXUwR72hFY4ZclFdmWLW2vM+8MLspsMSnCnDcoCjTyvUZiR
+MAXGswrIeN1m4k15PXVKuvB9ed+F9vrF83QsrufxV0ouDZmIBJlWCc17Ye8lVQR0
+Un1PZbILaNHEokmt8tAgHzI8Qjf5S7lJ9ROlcCrDBNclBy9WQskCAR0=
+-----END RSA PUBLIC KEY-----

src/questions/tests/fermat2.pem

@@ -0,0 +1,3 @@
+-----BEGIN RSA PUBLIC KEY-----
+MAgCAxCplQIBCg==
+-----END RSA PUBLIC KEY-----

src/questions/tests/test_fermat.test

@@ -0,0 +1,5 @@
+../../qa -a fermat fermat.pem > /dev/null
+[[ $? == 0 ]] || exit 1
+
+../../qa -a fermat fermat2.pem > /dev/null
+[[ $? == 0 ]] || exit 1