diff --git a/server/api/image.js b/server/api/image.js
index 276a419..56ab4c1 100644
--- a/server/api/image.js
+++ b/server/api/image.js
@@ -140,12 +140,17 @@ async function uploadImage (identifier, fields, files) {
     return bailOut(file, 'Avatars can only have an aspect ratio of 1:1')
   }
 
-  if (fields.scaleX) {
+  // Upscaling is not allowed
+  if ((fields.scaleX != null && fields.scaleX > 1) || (fields.scaleY != null && fields.scaleY > 1)) {
+    return bailOut(file, 'Image upscaling is not allowed.')
+  }
+
+  if (fields.scaleX != null) {
     fields.x *= fields.scaleX
     fields.width *= fields.scaleX
   }
 
-  if (fields.scaleY) {
+  if (fields.scaleY != null) {
     fields.y *= fields.scaleY
     fields.height *= fields.scaleY
   }