Esto te podría valer:
Código HTML:
<html>
<head>
<script>
function deshabilitarChecks(){
var numMarcados=0;
var i=0;
var checks=document.getElementsByName("check[]");
while (numMarcados<3 && i<checks.length){
if(checks[i].checked){
numMarcados++;
}
i++;
}
if (numMarcados>=3){
for (i=0;i<checks.length;i++){
if (!checks[i].checked){
checks[i].disabled=true;
}
}
}else{
for (i=0;i<checks.length;i++){
checks[i].disabled=false;
}
}
}
</script>
</head>
<body >
<form name="formulario">
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
<br>
<input type="checkbox" name="check[]" onClick="deshabilitarChecks()">
</form>
</body>
</html>