Fix BIP143 standardness rules for CHECKMULTISIG

From https://github.com/bitcoin/bips/blob/master/bip-0143.mediawiki#restrictions-on-public-key-type

> As a default policy, only compressed public keys are accepted in P2WPKH and
> P2WSH. Each public key passed to a sigop inside version 0 witness program must
> be a compressed key: the first byte MUST be either 0x02 or 0x03, and the size
> MUST be 33 bytes. Transactions that break this rule will not be relayed or mined
> by default.

PR bitcoin#8499 's implemenation is insufficent to meet BIP143's requirements as it
only checks those pubkeys processed by CHECKMULTISIG, whereas BIP143 requires
that every public key passed to the CHECKMULTISIG be validated.

Note: these restrictions are policy only and not consensus rules.

Author: roconnor-blockstream

src/script/interpreter.cpp

@@ -215,14 +215,10 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, script_ver
     return true;
 }
 
-bool static CheckPubKeyEncoding(const valtype &vchPubKey, script_verify_flags flags, const SigVersion &sigversion, ScriptError* serror) {
+bool static CheckPubKeyEncoding(const valtype &vchPubKey, script_verify_flags flags, ScriptError* serror) {
     if ((flags & SCRIPT_VERIFY_STRICTENC) != 0 && !IsCompressedOrUncompressedPubKey(vchPubKey)) {
         return set_error(serror, SCRIPT_ERR_PUBKEYTYPE);
     }
-    // Only compressed keys are accepted in segwit
-    if ((flags & SCRIPT_VERIFY_WITNESS_PUBKEYTYPE) != 0 && sigversion == SigVersion::WITNESS_V0 && !IsCompressedPubKey(vchPubKey)) {
-        return set_error(serror, SCRIPT_ERR_WITNESS_PUBKEYTYPE);
-    }
     return true;
 }
 
@@ -332,10 +328,16 @@ static bool EvalChecksigPreTapscript(const valtype& vchSig, const valtype& vchPu
             return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE);
     }
 
-    if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) {
+    if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, serror)) {
         //serror is set
         return false;
     }
+
+    if ((flags & SCRIPT_VERIFY_WITNESS_PUBKEYTYPE) != 0 && sigversion == SigVersion::WITNESS_V0 && !IsCompressedPubKey(vchPubKey)) {
+        // Only compressed keys are accepted in segwit
+        return set_error(serror, SCRIPT_ERR_WITNESS_PUBKEYTYPE);
+    }
+
     fSuccess = checker.CheckECDSASignature(vchSig, vchPubKey, scriptCode, sigversion);
 
     if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size())
@@ -1149,6 +1151,16 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript&
                         }
                     }
 
+                    if ((flags & SCRIPT_VERIFY_WITNESS_PUBKEYTYPE) != 0 && sigversion == SigVersion::WITNESS_V0) {
+                        for (int k = 0; k < nKeysCount; k++) {
+                            valtype& vchPubKey = stacktop(-ikey-k);
+                            if (!IsCompressedPubKey(vchPubKey)) {
+                                // Only compressed keys are accepted in segwit
+                                return set_error(serror, SCRIPT_ERR_WITNESS_PUBKEYTYPE);
+                            }
+                        }
+                    }
+
                     bool fSuccess = true;
                     while (fSuccess && nSigsCount > 0)
                     {
@@ -1158,7 +1170,7 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript&
                         // Note how this makes the exact order of pubkey/signature evaluation
                         // distinguishable by CHECKMULTISIG NOT if the STRICTENC flag is set.
                         // See the script_(in)valid tests for details.
-                        if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) {
+                        if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, serror)) {
                             // serror is set
                             return false;
                         }