Encryption and decryption error 0x0407106B using OpenSSL

Question

I am writing a C procedure that reads a base64 public key string and proceeds to encrypt the string. I also test the same string decryption, but when I try to decode, I get error 0x0407106B:

$ openssl errstr 0x0407106B error:0407106B:rsa routines:RSA_padding_check_PKCS1_type_2:block type is not 02 

Here is the code

 #include <stdio.h> #include <string.h> #include <stdlib.h> #include <ctype.h> #include <openssl/rsa.h> #include <openssl/engine.h> //#define PADDING RSA_PKCS1_OAEP_PADDING #define PADDING RSA_PKCS1_PADDING //#define PADDING RSA_NO_PADDING main() { // public key char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n"; // private key char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n"; // String to encrypt char *str = "1234"; ERR_load_crypto_strings(); BIO *bpo = BIO_new_mem_buf(b64_pKey, -1); RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL); if ( !pubKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } int rsa_length = RSA_size(pubKey); BIO *b64 = NULL; BIO *bmem = NULL; BUF_MEM *bptr = NULL; unsigned char encrypted[2560] = { 0 }; unsigned char retencrypted[2560] = { 0 }; int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING); if ( resultEncrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } /* * Show base 64 encrypted string */ b64 = BIO_new((BIO_METHOD *)BIO_f_base64()); BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL); bmem = BIO_new(BIO_s_mem()); b64 = BIO_push(b64, bmem); BIO_write(b64, encrypted, resultEncrypt); BIO_flush(b64); BIO_get_mem_ptr(b64, &bptr); memcpy(retencrypted, bptr->data, bptr->length); BIO_free(b64); BIO_free(bpo); RSA_free(pubKey); printf("Encrypted string:%s\n",retencrypted); /* * Now decrypt this very string with the private key */ BIO *bpop = BIO_new_mem_buf(b64priv_key, -1); RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL); if ( !privKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } rsa_length = RSA_size(privKey); unsigned char decrypted[2560] = { 0 }; int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING); if ( resultDecrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted); BIO_free(bpop); RSA_free(privKey); ERR_free_strings(); } KEY ----- \ nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w + 8HUdECo8V5yuKYrWJmUbL \ ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf / ZYQDWCK726u6 \ nhsZA81AblAOOXKaUaxvFC + ZKRJf + MtUGnv0v7CrGoblm1mMC / OQI1JfSsYi68Epn \ naOLepTZw + GLTnusQgwIDAQAB \ n ----- END PUBLIC KEY-- #include <stdio.h> #include <string.h> #include <stdlib.h> #include <ctype.h> #include <openssl/rsa.h> #include <openssl/engine.h> //#define PADDING RSA_PKCS1_OAEP_PADDING #define PADDING RSA_PKCS1_PADDING //#define PADDING RSA_NO_PADDING main() { // public key char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n"; // private key char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n"; // String to encrypt char *str = "1234"; ERR_load_crypto_strings(); BIO *bpo = BIO_new_mem_buf(b64_pKey, -1); RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL); if ( !pubKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } int rsa_length = RSA_size(pubKey); BIO *b64 = NULL; BIO *bmem = NULL; BUF_MEM *bptr = NULL; unsigned char encrypted[2560] = { 0 }; unsigned char retencrypted[2560] = { 0 }; int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING); if ( resultEncrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } /* * Show base 64 encrypted string */ b64 = BIO_new((BIO_METHOD *)BIO_f_base64()); BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL); bmem = BIO_new(BIO_s_mem()); b64 = BIO_push(b64, bmem); BIO_write(b64, encrypted, resultEncrypt); BIO_flush(b64); BIO_get_mem_ptr(b64, &bptr); memcpy(retencrypted, bptr->data, bptr->length); BIO_free(b64); BIO_free(bpo); RSA_free(pubKey); printf("Encrypted string:%s\n",retencrypted); /* * Now decrypt this very string with the private key */ BIO *bpop = BIO_new_mem_buf(b64priv_key, -1); RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL); if ( !privKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } rsa_length = RSA_size(privKey); unsigned char decrypted[2560] = { 0 }; int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING); if ( resultDecrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted); BIO_free(bpop); RSA_free(privKey); ERR_free_strings(); } PRIVATE KEY ----- \ nMIICXAIBAAKBgQCp2w + 8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW \ nNOGVkMsCo6W4hpl5eHv1p9Hqdcf / ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC + ZK \ nRJf + MtUGnv0v7CrGoblm1mMC / OQI1JfSsYi68EpnaOLepTZw + GLTnusQgwIDAQAB \ nAoGBAKDuq3PikblH / 9YS11AgwjwC ++ 7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM \ nROIWtac / nsNcJCnvOnUjK / c3NIAaGJcfRPiH #include <stdio.h> #include <string.h> #include <stdlib.h> #include <ctype.h> #include <openssl/rsa.h> #include <openssl/engine.h> //#define PADDING RSA_PKCS1_OAEP_PADDING #define PADDING RSA_PKCS1_PADDING //#define PADDING RSA_NO_PADDING main() { // public key char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n"; // private key char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n"; // String to encrypt char *str = "1234"; ERR_load_crypto_strings(); BIO *bpo = BIO_new_mem_buf(b64_pKey, -1); RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL); if ( !pubKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } int rsa_length = RSA_size(pubKey); BIO *b64 = NULL; BIO *bmem = NULL; BUF_MEM *bptr = NULL; unsigned char encrypted[2560] = { 0 }; unsigned char retencrypted[2560] = { 0 }; int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING); if ( resultEncrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } /* * Show base 64 encrypted string */ b64 = BIO_new((BIO_METHOD *)BIO_f_base64()); BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL); bmem = BIO_new(BIO_s_mem()); b64 = BIO_push(b64, bmem); BIO_write(b64, encrypted, resultEncrypt); BIO_flush(b64); BIO_get_mem_ptr(b64, &bptr); memcpy(retencrypted, bptr->data, bptr->length); BIO_free(b64); BIO_free(bpo); RSA_free(pubKey); printf("Encrypted string:%s\n",retencrypted); /* * Now decrypt this very string with the private key */ BIO *bpop = BIO_new_mem_buf(b64priv_key, -1); RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL); if ( !privKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } rsa_length = RSA_size(privKey); unsigned char decrypted[2560] = { 0 }; int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING); if ( resultDecrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted); BIO_free(bpop); RSA_free(privKey); ERR_free_strings(); } / yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8 \ nY / uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV + jB6 / gktwk1QaFsy7dCXn7w03 + k \ nxrjEGGN + kQJBAMuKf55lDtU9K2Js3YSStTZAXP + Hz7XpoLxmbWFyGvBx806WjgAD \ n624irwS + 0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG + \ njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe \ noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR + aGZgWcqjPBrwp5x08t43mHxeb4wW8 \ ndFZ6 + trnntO4TMxkA9ECQB + yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio \ nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0 = \ n --- #include <stdio.h> #include <string.h> #include <stdlib.h> #include <ctype.h> #include <openssl/rsa.h> #include <openssl/engine.h> //#define PADDING RSA_PKCS1_OAEP_PADDING #define PADDING RSA_PKCS1_PADDING //#define PADDING RSA_NO_PADDING main() { // public key char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n"; // private key char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n"; // String to encrypt char *str = "1234"; ERR_load_crypto_strings(); BIO *bpo = BIO_new_mem_buf(b64_pKey, -1); RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL); if ( !pubKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } int rsa_length = RSA_size(pubKey); BIO *b64 = NULL; BIO *bmem = NULL; BUF_MEM *bptr = NULL; unsigned char encrypted[2560] = { 0 }; unsigned char retencrypted[2560] = { 0 }; int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING); if ( resultEncrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } /* * Show base 64 encrypted string */ b64 = BIO_new((BIO_METHOD *)BIO_f_base64()); BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL); bmem = BIO_new(BIO_s_mem()); b64 = BIO_push(b64, bmem); BIO_write(b64, encrypted, resultEncrypt); BIO_flush(b64); BIO_get_mem_ptr(b64, &bptr); memcpy(retencrypted, bptr->data, bptr->length); BIO_free(b64); BIO_free(bpo); RSA_free(pubKey); printf("Encrypted string:%s\n",retencrypted); /* * Now decrypt this very string with the private key */ BIO *bpop = BIO_new_mem_buf(b64priv_key, -1); RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL); if ( !privKey ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } rsa_length = RSA_size(privKey); unsigned char decrypted[2560] = { 0 }; int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING); if ( resultDecrypt == -1 ) { printf("%s\n", ERR_error_string(ERR_get_error(), NULL)); return; } printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted); BIO_free(bpop); RSA_free(privKey); ERR_free_strings(); } 

Note. I exported the private key using

 openssl rsa -in rsa_privatekey.pem -check 

and public key:

 openssl rsa -in rsa_privatekey.pem -pubout 

Why am I getting an error message?

Answer 1

The problem is that you are trying to decrypt the result with base64 encoding. You should try to decrypt the encryption result.

That is, instead of:

 int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING); 

You must call:

 int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), encrypted, decrypted, privKey, PADDING); 

In addition, there is a problem in the encryption call:

 int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING); 

Why are you passing PADDING as flen ? This should be the length of the string (i.e. 4 or 5, depending on whether you want to encrypt the null character).

If you want to write an encrypted string as ASCII (encoded with base64), that would be fine. But you must decode it before decrypting it.

Answer 2

The error you get is the block type is not 02 .

Although Omri is correct that you are transmitting incorrect data and you are going to encrypt only 1 byte, the error is because sizeof( encrypted ) is way too large (2560). In other words, your data receiver for RSA_public_encrypt must be a regular pointer unsigned char* , not unsigned char[2560] .

Where are you

 unsigned char encrypted[2560] = { 0 }; //X 2560?? RSA_public_encrypt fails. 

You have to use

 unsigned char *encrypted = (unsigned char*)malloc( rsa_length ) ; RSA_public_encrypt( DATALEN, (const unsigned char*)str, encrypted, pubKey, PADDING ) ; 

Note the error Omri pointed out that you used PADDING as the first arg argument to RSA_public_encrypt , while the data length is DATALEN .

If you fix this, you will get a similar error later with the decryption of the secret key. Correct it and you're on your way.